diff --git "a/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/logs/debug-internal.log" "b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/logs/debug-internal.log" @@ -0,0 +1,43317 @@ +2024-03-12 14:29:03,635 INFO StreamThr :2491270 [internal.py:wandb_internal():86] W&B internal server running at pid: 2491270, started at: 2024-03-12 14:29:03.634346 +2024-03-12 14:29:03,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status +2024-03-12 14:29:03,649 INFO WriterThread:2491270 [datastore.py:open_for_write():87] open: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/run-yrs4wcl6.wandb +2024-03-12 14:29:03,651 DEBUG SenderThread:2491270 [sender.py:send():382] send: header +2024-03-12 14:29:03,676 DEBUG SenderThread:2491270 [sender.py:send():382] send: run +2024-03-12 14:29:07,863 INFO SenderThread:2491270 [dir_watcher.py:__init__():211] watching files in: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files +2024-03-12 14:29:07,863 INFO SenderThread:2491270 [sender.py:_start_run_threads():1136] run started: yrs4wcl6 with start time 1710253743.636872 +2024-03-12 14:29:07,875 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: check_version +2024-03-12 14:29:07,875 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: check_version +2024-03-12 14:29:07,930 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: run_start +2024-03-12 14:29:07,941 DEBUG HandlerThread:2491270 [system_info.py:__init__():27] System info init +2024-03-12 14:29:07,941 DEBUG HandlerThread:2491270 [system_info.py:__init__():42] System info init done +2024-03-12 14:29:07,941 INFO HandlerThread:2491270 [system_monitor.py:start():194] Starting system monitor +2024-03-12 14:29:07,941 INFO SystemMonitor:2491270 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-12 14:29:07,942 INFO HandlerThread:2491270 [system_monitor.py:probe():214] Collecting system info +2024-03-12 14:29:07,943 INFO SystemMonitor:2491270 [interfaces.py:start():190] Started cpu monitoring +2024-03-12 14:29:07,943 INFO SystemMonitor:2491270 [interfaces.py:start():190] Started disk monitoring +2024-03-12 14:29:07,943 INFO SystemMonitor:2491270 [interfaces.py:start():190] Started gpu monitoring +2024-03-12 14:29:07,944 INFO SystemMonitor:2491270 [interfaces.py:start():190] Started memory monitoring +2024-03-12 14:29:07,944 INFO SystemMonitor:2491270 [interfaces.py:start():190] Started network monitoring +2024-03-12 14:29:08,013 DEBUG HandlerThread:2491270 [system_info.py:probe():151] Probing system +2024-03-12 14:29:08,017 DEBUG HandlerThread:2491270 [system_info.py:_probe_git():136] Probing git +2024-03-12 14:29:08,037 DEBUG HandlerThread:2491270 [system_info.py:_probe_git():144] Probing git done +2024-03-12 14:29:08,037 DEBUG HandlerThread:2491270 [system_info.py:probe():199] Probing system done +2024-03-12 14:29:08,037 DEBUG HandlerThread:2491270 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1027-oracle-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-03-12T14:29:08.013324', 'startedAt': '2024-03-12T14:29:03.589151', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'one-stage+7b_qformer2_256', '--run_id', 'qformer2_256', '--model.arch_specifier', 'qformer2_256'), 'state': 'running', 'program': '/home/lilei/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms.git', 'commit': 'b1c33c284c6e76d908c29e5545b00f5b8b4c601f'}, 'email': 'tobiaslee@qq.com', 'root': '/home/lilei/prismatic-vlms', 'host': 'compute-permanent-node-824', 'username': 'lilei', 'executable': '/home/lilei/anaconda3/envs/pvlm/bin/python', 'cpu_count': 128, 'cpu_count_logical': 255, 'cpu_freq': {'current': 2578.3530274509803, 'min': 1500.0, 'max': 2550.0}, 'cpu_freq_per_core': [{'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.171, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.526, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3250.052, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.918, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.534, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.538, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.553, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.537, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.359, 'min': 1500.0, 'max': 2550.0}, {'current': 3072.53, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.242, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.221, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}], 'disk': {'/': {'total': 242.1086883544922, 'used': 50.49263000488281}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 2015.6942749023438}} +2024-03-12 14:29:08,037 INFO HandlerThread:2491270 [system_monitor.py:probe():224] Finished collecting system info +2024-03-12 14:29:08,037 INFO HandlerThread:2491270 [system_monitor.py:probe():227] Publishing system info +2024-03-12 14:29:08,038 DEBUG HandlerThread:2491270 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment +2024-03-12 14:29:08,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/conda-environment.yaml +2024-03-12 14:29:13,095 DEBUG HandlerThread:2491270 [system_info.py:_save_conda():220] Saving conda packages done +2024-03-12 14:29:13,103 INFO HandlerThread:2491270 [system_monitor.py:probe():229] Finished publishing system info +2024-03-12 14:29:13,108 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:13,108 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 14:29:13,109 DEBUG SenderThread:2491270 [sender.py:send():382] send: files +2024-03-12 14:29:13,109 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-12 14:29:13,119 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: python_packages +2024-03-12 14:29:13,119 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: python_packages +2024-03-12 14:29:13,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:13,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:13,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:13,426 DEBUG SenderThread:2491270 [sender.py:send():382] send: telemetry +2024-03-12 14:29:13,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/conda-environment.yaml +2024-03-12 14:29:13,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-metadata.json +2024-03-12 14:29:13,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/requirements.txt +2024-03-12 14:29:13,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:29:14,066 INFO wandb-upload_0:2491270 [upload_job.py:push():131] Uploaded file /tmp/tmpa2oem8qgwandb/ijnknfbx-wandb-metadata.json +2024-03-12 14:29:14,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:15,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:29:18,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:29:19,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:24,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:28,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:28,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:28,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:30,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:35,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:35,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/config.yaml +2024-03-12 14:29:39,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:40,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:29:41,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:43,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:43,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:43,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:47,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:47,106 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:29:47,106 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:47,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:47,111 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:47,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:29:48,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:29:51,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:51,329 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:29:51,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:51,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:51,905 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:29:52,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:52,908 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:29:55,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:55,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:29:55,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:55,733 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:55,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:29:56,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:29:57,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:58,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:58,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:58,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:01,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:01,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:01,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:01,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:01,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:02,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:03,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:06,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:06,059 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:06,059 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:06,064 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:06,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:06,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:07,944 DEBUG SystemMonitor:2491270 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-12 14:30:07,947 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:30:08,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:10,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:10,970 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:10,971 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:10,975 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:11,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:12,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:13,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:13,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:13,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:14,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:17,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:17,116 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:17,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:17,121 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:17,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:18,936 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:20,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:22,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:22,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:22,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:22,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:22,938 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:24,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:26,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:28,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:28,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:28,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:29,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:29,593 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:29,593 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:29,598 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:29,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:30,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:31,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:34,244 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:34,245 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:34,245 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:34,250 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:34,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:36,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:37,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:37,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:30:39,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:39,051 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:39,051 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:39,055 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:39,953 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:40,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:43,056 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:43,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:43,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:43,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:43,852 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:43,853 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:43,853 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:43,858 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:43,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:44,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:48,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:48,249 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:48,250 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:48,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:48,254 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:48,962 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:50,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:53,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:54,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:54,116 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:54,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:54,121 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:54,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:56,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:58,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:58,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:58,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:58,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:00,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:00,124 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:00,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:00,129 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:00,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:02,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:04,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:05,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:05,102 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:05,103 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:05,107 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:05,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:06,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:07,952 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:31:09,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:10,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:10,495 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:10,496 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:10,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:10,985 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:12,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:13,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:13,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:13,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:14,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:14,968 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:14,969 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:14,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:14,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:14,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:16,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:19,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:19,201 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:19,202 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:19,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:19,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:20,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:20,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:25,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:25,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:25,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:25,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:25,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:26,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:27,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:28,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:28,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:28,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:30,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:30,162 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:30,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:30,166 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:31,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:31,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:33,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:36,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:37,954 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:31:38,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:38,412 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:38,413 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:38,418 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:39,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:41,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:41,419 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:42,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:42,685 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:42,685 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:42,690 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:43,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:43,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:43,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:43,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:45,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:47,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:47,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:47,621 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:47,622 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:47,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:48,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:49,025 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:52,667 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:56,842 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:56,843 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:56,843 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:56,848 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:57,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:57,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:58,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:58,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:58,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:59,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:00,860 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:00,861 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:00,861 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:00,866 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:01,035 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:02,902 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:03,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:05,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:05,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:05,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:05,161 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:06,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:07,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:07,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:32:07,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:10,939 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:10,940 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:10,940 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:10,946 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:11,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:13,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:13,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:13,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:13,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:16,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:16,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:16,820 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:16,825 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:17,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:18,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:19,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:21,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:21,900 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:21,900 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:21,905 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:22,057 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:23,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:23,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:28,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:28,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:28,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:28,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:28,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:28,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:28,740 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:29,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:29,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:31,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:34,078 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:34,079 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:34,079 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:34,084 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:35,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:35,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:35,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:37,958 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:32:38,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:38,803 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:38,803 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:38,809 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:39,073 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:40,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:41,077 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:43,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:43,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:43,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:43,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:43,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:43,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:43,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:44,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:45,081 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:45,898 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:50,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:50,188 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:50,188 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:50,194 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:51,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:51,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:51,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:56,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:58,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:58,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:58,033 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:58,038 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:58,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:58,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:58,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:58,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:59,092 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:01,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:02,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:02,338 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:02,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:02,343 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:03,096 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:05,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:06,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:06,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:06,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:06,655 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:06,659 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:07,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:07,966 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:33:09,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:11,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:12,823 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:12,824 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:12,824 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:12,830 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:13,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:13,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:13,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:13,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:15,109 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:16,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:16,632 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:16,632 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:16,637 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:17,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:17,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:19,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:20,924 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:20,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:20,925 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:20,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:21,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:22,930 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:23,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:26,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:26,683 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:26,683 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:26,688 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:27,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:28,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:28,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:28,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:28,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:29,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:32,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:32,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:32,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:32,159 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:33,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:33,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:34,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:37,963 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:33:39,964 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:40,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:40,597 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:40,598 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:40,602 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:41,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:43,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:43,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:43,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:43,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:45,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:50,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:50,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:50,692 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:50,692 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:50,697 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:51,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:53,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:55,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:56,225 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:56,226 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:56,226 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:56,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:57,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:57,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:58,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:58,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:58,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:01,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:02,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:02,541 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:02,541 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:02,546 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:03,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:05,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:06,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:07,320 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:07,320 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:07,321 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:07,325 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:07,965 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:34:08,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:09,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:11,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:12,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:12,295 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:12,295 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:12,300 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:13,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:13,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:13,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:13,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:13,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:16,903 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:16,904 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:16,904 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:16,908 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:17,173 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:17,946 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:19,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:22,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:22,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:22,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:22,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:23,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:23,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:25,182 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:28,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:28,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:28,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:29,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:29,014 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:29,014 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:29,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:29,019 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:29,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:31,187 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:34,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:34,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:34,620 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:34,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:34,625 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:35,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:37,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:37,972 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:34:39,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:40,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:40,646 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:40,646 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:40,651 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:41,196 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:43,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:43,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:43,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:43,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:45,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:47,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:47,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:47,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:47,132 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:47,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:49,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:51,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:51,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:51,792 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:51,793 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:51,797 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:52,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:53,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:56,798 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:58,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:58,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:58,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:59,259 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:59,259 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:59,260 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:59,264 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:00,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:01,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:02,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:05,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:05,630 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:05,631 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:05,635 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:06,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:07,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:07,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:07,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:35:10,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:10,105 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:10,106 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:10,110 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:10,225 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:11,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:13,111 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:13,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:13,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:13,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:15,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:15,141 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:15,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:15,147 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:15,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:17,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:18,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:19,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:19,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:19,501 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:19,506 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:20,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:21,238 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:23,507 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:24,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:24,105 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:24,106 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:24,110 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:24,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:25,242 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:28,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:28,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:28,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:29,320 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:32,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:32,197 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:32,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:32,202 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:32,246 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:33,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:35,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:37,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:37,703 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:37,703 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:37,708 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:37,977 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:35:38,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:39,256 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:40,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:42,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:42,658 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:42,659 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:42,664 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:43,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:43,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:43,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:43,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:45,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:46,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:46,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:46,927 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:46,927 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:46,932 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:47,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:49,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:51,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:52,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:52,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:52,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:52,641 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:53,269 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:55,272 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:56,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:56,890 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:56,890 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:56,895 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:57,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:57,895 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:58,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:58,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:58,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:59,276 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:01,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:01,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:01,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:01,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:02,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:03,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:03,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:05,958 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:05,958 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:05,959 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:05,963 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:06,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:07,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:07,973 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:36:08,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:10,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:10,090 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:10,091 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:10,095 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:10,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:11,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:13,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:13,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:13,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:14,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:14,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:14,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:14,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:14,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:15,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:17,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:19,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:19,124 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:19,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:19,130 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:19,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:20,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:21,301 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:24,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:24,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:24,436 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:24,441 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:25,303 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:25,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:25,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:28,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:28,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:28,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:29,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:29,731 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:29,731 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:29,736 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:30,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:30,774 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:31,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:33,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:33,819 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:33,820 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:33,825 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:34,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:35,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:35,825 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:37,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:36:37,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:37,997 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:37,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:38,002 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:38,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:39,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:41,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:42,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:42,496 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:42,496 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:42,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:43,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:43,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:43,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:43,323 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:43,323 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:46,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:48,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:48,647 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:48,647 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:48,652 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:49,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:51,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:51,690 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:54,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:54,152 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:54,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:54,157 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:54,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:55,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:57,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:58,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:58,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:58,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:58,574 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:58,575 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:58,575 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:58,580 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:59,336 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:01,341 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:02,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:04,930 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:04,931 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:04,931 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:04,936 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:05,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:07,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:07,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:07,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:37:10,804 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:10,805 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:10,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:10,810 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:11,349 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:13,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:13,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:13,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:13,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:13,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:15,320 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:15,321 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:15,321 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:15,326 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:15,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:17,357 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:19,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:24,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:24,431 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:24,431 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:24,432 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:24,436 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:25,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:25,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:28,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:28,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:28,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:28,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:28,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:28,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:28,923 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:29,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:29,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:31,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:34,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:35,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:35,297 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:35,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:35,303 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:35,372 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:37,376 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:37,980 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:37:39,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:39,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:39,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:39,824 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:40,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:40,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:41,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:43,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:43,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:43,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:46,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:46,228 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:46,228 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:46,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:46,233 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:46,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:47,386 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:51,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:52,176 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:52,177 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:52,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:52,183 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:52,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:53,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:56,851 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:56,852 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:56,853 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:56,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:56,858 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:57,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:58,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:58,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:58,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:59,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:01,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:01,882 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:01,883 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:01,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:01,888 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:02,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:03,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:05,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:05,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:05,969 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:05,975 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:06,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:06,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:07,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:07,988 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:38:10,519 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:10,520 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:10,520 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:10,525 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:11,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:11,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:12,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:13,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:13,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:13,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:16,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:16,444 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:16,445 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:16,450 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:17,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:17,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:18,486 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:21,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:21,696 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:21,696 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:21,701 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:22,422 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:23,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:23,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:26,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:26,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:26,187 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:26,191 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:26,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:27,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:28,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:28,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:28,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:29,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:34,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:34,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:34,430 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:34,430 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:34,435 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:35,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:35,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:37,984 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:38:39,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:39,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:39,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:39,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:39,446 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:40,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:41,445 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:43,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:43,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:43,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:45,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:45,237 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:45,237 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:45,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:45,242 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:45,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:47,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:49,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:49,225 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:49,225 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:49,230 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:49,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:51,230 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:51,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:56,231 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:57,639 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:57,640 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:57,640 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:57,644 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:58,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:58,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:58,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:58,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:59,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:01,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:01,946 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:01,947 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:01,947 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:01,952 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:02,465 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:03,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:05,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:05,858 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:05,858 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:05,863 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:06,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:06,864 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:07,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:07,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:39:10,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:10,229 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:10,229 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:10,234 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:10,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:11,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:12,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:13,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:13,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:13,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:14,688 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:14,688 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:14,689 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:14,694 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:15,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:17,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:17,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:19,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:19,381 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:19,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:19,386 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:19,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:21,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:23,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:23,788 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:23,788 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:23,789 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:23,794 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:24,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:25,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:28,062 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:28,063 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:28,063 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:28,068 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:28,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:28,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:28,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:28,415 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:28,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:29,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:33,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:34,264 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:34,265 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:34,265 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:34,270 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:34,501 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:35,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:37,989 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:39:38,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:40,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:40,817 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:40,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:40,823 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:41,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:43,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:43,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:43,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:43,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:44,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:46,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:46,740 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:46,741 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:46,746 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:47,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:49,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:49,782 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:54,783 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:55,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:55,310 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:55,310 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:55,315 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:55,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:57,526 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:58,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:58,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:58,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:00,424 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:00,496 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:00,497 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:00,497 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:00,503 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:00,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:01,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:05,539 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:07,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:07,352 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:07,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:07,358 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:07,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:07,991 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:40:09,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:10,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:12,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:12,420 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:12,421 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:12,427 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:12,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:13,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:13,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:13,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:13,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:16,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:16,850 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:16,851 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:16,851 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:16,857 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:17,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:19,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:21,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:21,427 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:21,428 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:21,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:21,433 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:21,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:23,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:26,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:26,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:26,557 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:26,557 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:26,562 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:27,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:27,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:28,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:28,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:28,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:32,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:33,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:33,194 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:33,194 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:33,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:33,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:35,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:37,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:37,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:37,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:37,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:37,625 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:37,994 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:40:38,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:39,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:42,091 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:42,092 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:42,092 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:42,103 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:42,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:43,104 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:43,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:43,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:43,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:43,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:46,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:46,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:46,897 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:46,906 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:47,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:48,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:49,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:51,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:51,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:51,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:51,653 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:52,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:53,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:54,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:58,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:58,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:58,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:58,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:58,666 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:58,666 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:58,677 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:59,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:59,617 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:59,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:03,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:03,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:03,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:03,019 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:03,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:05,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:05,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:07,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:07,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:07,703 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:07,710 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:07,996 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:41:08,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:09,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:10,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:12,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:12,213 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:12,213 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:12,223 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:12,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:13,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:13,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:13,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:13,645 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:16,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:17,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:17,782 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:17,782 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:17,789 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:18,650 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:19,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:21,827 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:22,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:22,016 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:22,017 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:22,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:22,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:23,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:26,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:26,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:26,458 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:26,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:26,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:27,471 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:27,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:28,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:28,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:28,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:33,062 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:33,063 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:33,063 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:33,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:33,071 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:33,680 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:35,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:37,504 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:37,505 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:37,505 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:37,514 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:37,692 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:38,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:41:39,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:39,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:42,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:42,588 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:42,588 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:42,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:42,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:43,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:43,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:43,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:43,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:44,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:46,884 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:46,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:46,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:46,897 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:47,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:49,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:49,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:52,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:52,171 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:52,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:52,181 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:52,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:53,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:55,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:56,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:56,017 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:56,017 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:56,024 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:56,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:57,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:58,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:58,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:58,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:00,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:00,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:00,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:00,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:00,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:01,733 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:03,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:04,919 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:04,920 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:04,920 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:04,928 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:05,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:05,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:07,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:08,000 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:42:08,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:08,859 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:08,859 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:08,865 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:09,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:09,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:11,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:13,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:13,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:13,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:17,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:18,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:18,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:18,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:18,459 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:18,760 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:19,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:22,491 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:26,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:26,190 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:26,190 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:26,203 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:26,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:27,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:28,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:28,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:28,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:33,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:34,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:34,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:34,639 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:34,646 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:34,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:35,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:38,002 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:42:39,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:43,047 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:43,048 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:43,049 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:43,062 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:43,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:43,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:43,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:43,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:44,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:45,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:47,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:47,193 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:47,193 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:47,205 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:47,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:49,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:50,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:52,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:52,463 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:52,463 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:52,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:52,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:53,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:55,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:57,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:57,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:57,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:57,032 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:57,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:58,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:58,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:58,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:59,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:01,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:01,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:01,705 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:01,705 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:01,714 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:01,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:03,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:06,747 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:07,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:07,128 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:07,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:07,135 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:07,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:08,005 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:43:09,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:12,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:12,178 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:12,179 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:12,180 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:12,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:12,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:13,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:13,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:13,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:13,847 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:17,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:17,115 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:17,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:17,116 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:17,128 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:17,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:19,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:22,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:23,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:23,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:23,246 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:23,252 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:23,860 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:25,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:27,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:27,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:27,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:27,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:27,896 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:28,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:28,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:28,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:28,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:29,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:32,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:32,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:32,990 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:32,991 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:32,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:33,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:35,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:38,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:43:38,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:41,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:41,493 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:41,494 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:41,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:41,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:43,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:43,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:43,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:43,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:43,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:46,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:46,016 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:46,017 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:46,025 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:46,902 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:47,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:49,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:50,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:50,795 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:50,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:50,805 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:50,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:51,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:54,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:54,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:54,862 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:54,863 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:54,871 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:54,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:55,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:58,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:58,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:58,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:00,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:00,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:00,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:00,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:00,357 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:00,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:01,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:05,225 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:05,226 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:05,226 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:05,233 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:05,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:06,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:07,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:08,019 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:44:09,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:09,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:09,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:09,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:09,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:11,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:11,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:13,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:13,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:13,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:14,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:14,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:14,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:14,862 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:14,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:15,954 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:16,898 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:20,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:20,923 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:20,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:20,931 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:20,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:21,932 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:21,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:25,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:25,811 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:25,811 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:25,818 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:25,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:27,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:27,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:28,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:28,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:28,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:30,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:30,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:30,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:30,377 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:30,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:31,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:33,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:34,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:34,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:34,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:34,896 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:34,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:35,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:38,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:44:39,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:39,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:39,594 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:39,595 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:39,600 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:39,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:41,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:43,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:43,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:43,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:44,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:44,059 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:44,059 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:44,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:44,065 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:44,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:46,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:49,103 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:52,191 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:52,192 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:52,192 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:52,201 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:53,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:54,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:54,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:57,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:57,694 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:57,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:57,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:58,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:58,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:58,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:58,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:59,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:00,024 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:03,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:03,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:03,498 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:03,504 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:04,029 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:04,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:06,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:07,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:07,889 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:07,889 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:07,895 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:08,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:45:08,035 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:10,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:10,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:13,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:13,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:13,059 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:13,065 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:13,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:13,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:13,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:14,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:14,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:15,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:17,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:17,360 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:17,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:17,365 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:18,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:20,051 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:21,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:25,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:25,765 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:25,765 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:25,771 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:26,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:26,772 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:28,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:28,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:28,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:28,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:31,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:31,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:31,612 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:31,618 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:32,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:32,654 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:34,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:36,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:36,791 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:36,791 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:36,796 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:37,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:37,797 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:38,016 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:45:38,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:41,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:41,963 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:41,964 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:41,968 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:42,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:42,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:43,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:43,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:43,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:44,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:47,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:47,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:47,767 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:47,772 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:48,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:48,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:50,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:52,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:52,579 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:52,579 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:52,584 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:53,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:54,086 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:54,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:58,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:58,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:58,038 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:58,044 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:58,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:58,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:58,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:58,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:59,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:00,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:01,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:01,841 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:01,842 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:01,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:02,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:04,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:04,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:06,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:06,609 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:06,610 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:06,615 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:07,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:08,018 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:46:08,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:10,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:10,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:10,961 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:10,961 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:10,966 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:11,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:12,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:13,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:13,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:15,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:17,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:17,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:17,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:17,173 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:18,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:20,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:21,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:21,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:21,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:21,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:21,739 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:22,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:24,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:26,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:26,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:26,013 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:26,018 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:26,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:27,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:28,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:28,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:28,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:28,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:32,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:32,374 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:32,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:32,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:32,379 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:33,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:34,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:37,419 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:37,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:37,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:37,522 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:37,527 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:38,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:46:38,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:40,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:42,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:42,758 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:42,758 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:42,759 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:42,763 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:43,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:43,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:43,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:43,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:44,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:48,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:48,254 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:48,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:48,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:48,259 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:49,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:50,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:52,934 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:52,935 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:52,935 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:52,940 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:53,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:53,940 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:54,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:57,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:57,772 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:57,772 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:57,777 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:58,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:58,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:58,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:58,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:59,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:00,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:02,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:02,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:02,871 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:02,876 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:03,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:04,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:04,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:07,986 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:07,988 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:07,988 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:07,993 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:08,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:47:08,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:10,024 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:10,169 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:13,108 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:13,109 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:13,110 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:13,115 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:13,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:13,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:13,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:13,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:14,173 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:15,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:17,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:17,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:17,448 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:17,453 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:18,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:20,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:20,491 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:22,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:22,670 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:22,671 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:22,676 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:23,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:24,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:25,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:27,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:27,258 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:27,259 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:27,263 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:28,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:28,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:28,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:28,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:30,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:31,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:32,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:32,379 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:32,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:32,385 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:33,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:34,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:36,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:37,220 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:37,221 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:37,221 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:37,226 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:38,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:47:38,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:38,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:41,904 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:41,905 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:41,905 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:41,906 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:41,910 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:42,202 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:43,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:43,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:43,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:44,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:46,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:46,352 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:46,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:46,358 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:47,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:47,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:48,210 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:52,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:55,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:55,392 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:55,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:55,398 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:56,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:57,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:58,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:58,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:58,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:00,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:00,811 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:00,811 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:00,816 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:01,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:02,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:02,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:07,855 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:08,027 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:48:08,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:08,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:08,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:08,181 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:08,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:10,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:12,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:12,478 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:12,478 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:12,483 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:13,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:13,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:13,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:13,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:13,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:14,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:17,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:17,281 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:17,281 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:17,286 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:18,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:19,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:20,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:24,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:25,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:25,555 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:25,555 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:25,560 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:26,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:28,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:28,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:28,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:28,251 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:29,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:30,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:30,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:30,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:30,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:31,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:32,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:34,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:36,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:36,464 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:36,464 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:36,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:37,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:38,035 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:48:38,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:40,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:41,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:41,411 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:41,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:41,416 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:42,264 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:43,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:43,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:43,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:44,267 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:45,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:48,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:48,169 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:48,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:48,174 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:48,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:50,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:51,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:52,668 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:52,669 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:52,669 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:52,674 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:53,276 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:54,279 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:56,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:58,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:58,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:58,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:58,830 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:58,830 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:58,835 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:59,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:00,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:01,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:03,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:03,084 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:03,084 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:03,089 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:03,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:04,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:07,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:08,031 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:49:08,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:08,138 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:08,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:08,144 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:08,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:10,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:12,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:12,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:12,598 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:12,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:12,603 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:13,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:13,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:14,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:16,911 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:16,911 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:16,912 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:16,916 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:17,301 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:17,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:18,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:22,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:22,204 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:22,205 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:22,210 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:22,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:23,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:24,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:27,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:27,182 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:27,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:27,188 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:27,311 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:28,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:28,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:28,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:28,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:28,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:31,876 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:31,877 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:31,877 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:31,882 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:32,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:33,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:34,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:38,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:49:39,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:40,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:40,304 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:40,304 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:40,310 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:40,323 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:42,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:43,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:43,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:43,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:44,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:45,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:45,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:45,535 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:45,540 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:46,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:48,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:49,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:49,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:49,629 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:49,630 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:49,634 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:50,333 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:52,336 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:53,936 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:53,937 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:53,937 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:53,942 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:54,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:54,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:56,341 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:58,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:58,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:58,332 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:58,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:58,337 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:58,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:00,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:00,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:03,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:03,237 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:03,237 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:03,242 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:03,346 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:04,349 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:06,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:07,518 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:07,518 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:07,519 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:07,523 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:08,036 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:50:08,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:10,355 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:12,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:12,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:12,152 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:12,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:12,158 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:12,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:13,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:14,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:16,842 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:16,842 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:16,842 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:16,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:17,362 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:17,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:18,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:22,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:25,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:25,159 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:25,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:25,164 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:25,368 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:26,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:28,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:28,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:28,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:28,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:31,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:31,927 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:31,927 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:31,932 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:32,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:33,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:34,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:38,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:50:39,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:41,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:41,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:41,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:41,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:42,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:43,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:43,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:43,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:44,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:44,385 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:46,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:46,296 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:46,296 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:46,301 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:46,386 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:48,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:50,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:54,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:54,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:54,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:54,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:55,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:55,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:56,395 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:58,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:59,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:59,740 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:59,741 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:59,745 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:00,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:01,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:02,402 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:04,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:04,662 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:04,662 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:04,667 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:05,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:06,407 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:07,667 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:08,043 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:51:09,523 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:09,524 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:09,524 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:09,529 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:10,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:12,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:13,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:13,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:13,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:13,803 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:13,803 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:13,808 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:14,415 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:16,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:18,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:18,653 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:18,653 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:18,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:18,658 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:19,420 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:20,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:23,659 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:25,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:25,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:25,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:25,819 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:26,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:28,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:28,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:28,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:28,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:29,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:30,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:30,102 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:30,102 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:30,107 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:30,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:32,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:34,875 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:34,875 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:34,876 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:34,876 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:34,881 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:35,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:36,438 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:38,042 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:51:40,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:40,424 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:40,425 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:40,425 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:40,430 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:40,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:42,445 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:43,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:43,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:43,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:45,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:45,855 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:45,856 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:45,856 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:45,861 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:46,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:48,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:50,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:52,042 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:52,042 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:52,043 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:52,047 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:52,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:54,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:56,048 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:56,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:56,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:56,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:56,177 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:56,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:58,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:58,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:00,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:00,598 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:00,598 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:00,603 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:01,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:01,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:02,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:06,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:06,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:06,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:06,885 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:06,889 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:07,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:08,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:52:08,472 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:11,532 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:11,532 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:11,533 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:11,537 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:12,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:12,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:12,538 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:13,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:15,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:15,649 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:15,649 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:15,654 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:16,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:17,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:18,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:20,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:20,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:20,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:20,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:20,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:22,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:23,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:28,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:28,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:28,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:28,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:28,471 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:28,471 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:28,471 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:28,477 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:28,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:30,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:33,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:33,189 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:33,189 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:33,194 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:33,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:34,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:34,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:37,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:37,651 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:37,651 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:37,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:38,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:52:38,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:40,048 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:40,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:43,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:43,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:43,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:43,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:43,330 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:43,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:43,335 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:43,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:44,508 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:45,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:47,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:47,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:47,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:47,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:48,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:50,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:50,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:54,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:54,662 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:54,663 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:54,668 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:55,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:56,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:56,668 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:58,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:58,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:58,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:59,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:59,137 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:59,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:59,142 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:59,521 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:00,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:02,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:04,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:04,956 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:04,956 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:04,961 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:05,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:06,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:07,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:08,049 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:53:09,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:09,879 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:09,879 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:09,884 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:10,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:12,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:13,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:13,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:13,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:16,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:16,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:16,013 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:16,018 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:16,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:18,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:19,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:20,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:20,863 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:20,863 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:20,868 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:21,545 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:22,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:24,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:26,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:26,138 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:26,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:26,143 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:26,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:28,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:28,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:28,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:28,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:30,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:30,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:30,658 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:30,658 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:30,663 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:31,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:32,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:35,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:35,153 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:35,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:35,159 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:35,560 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:36,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:36,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:38,055 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:53:39,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:39,942 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:39,942 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:39,947 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:40,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:41,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:42,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:43,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:43,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:43,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:44,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:44,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:44,976 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:44,981 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:45,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:46,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:47,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:51,488 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:51,489 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:51,489 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:51,494 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:51,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:52,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:52,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:56,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:56,423 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:56,423 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:56,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:56,581 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:58,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:58,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:58,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:58,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:58,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:01,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:01,302 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:01,302 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:01,307 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:01,588 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:02,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:03,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:06,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:06,386 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:06,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:06,391 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:06,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:08,053 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:54:08,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:09,055 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:11,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:11,226 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:11,227 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:11,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:11,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:12,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:13,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:13,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:13,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:14,415 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:19,415 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:19,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:19,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:19,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:19,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:20,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:22,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:24,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:24,572 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:24,572 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:24,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:24,577 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:24,612 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:26,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:28,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:28,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:28,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:29,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:29,021 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:29,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:29,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:29,617 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:30,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:30,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:34,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:34,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:34,791 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:34,796 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:35,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:35,796 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:36,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:38,059 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:54:39,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:39,713 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:39,713 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:39,719 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:40,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:41,719 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:42,633 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:43,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:43,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:43,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:43,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:43,910 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:43,910 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:43,915 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:44,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:46,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:46,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:48,446 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:48,447 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:48,447 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:48,452 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:48,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:50,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:52,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:53,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:53,251 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:53,252 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:53,256 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:53,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:54,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:57,515 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:57,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:57,516 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:57,516 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:57,520 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:57,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:58,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:58,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:58,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:58,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:03,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:04,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:04,290 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:04,290 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:04,295 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:04,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:06,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:08,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:55:09,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:10,891 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:10,892 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:10,892 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:10,897 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:11,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:12,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:13,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:13,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:13,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:14,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:15,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:15,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:15,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:15,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:16,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:18,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:20,033 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:22,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:22,388 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:22,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:22,394 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:22,675 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:24,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:25,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:27,747 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:27,747 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:27,748 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:27,753 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:28,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:28,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:28,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:28,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:28,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:30,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:33,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:33,717 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:33,717 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:33,723 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:34,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:34,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:35,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:38,060 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:55:38,078 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:38,078 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:38,079 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:38,084 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:38,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:40,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:41,084 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:42,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:42,671 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:42,672 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:42,677 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:42,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:43,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:43,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:43,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:44,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:46,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:47,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:47,067 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:47,067 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:47,072 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:47,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:48,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:52,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:52,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:52,408 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:52,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:52,413 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:52,707 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:54,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:57,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:57,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:57,756 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:57,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:57,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:58,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:58,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:58,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:58,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:58,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:56:02,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:03,064 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:03,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:56:03,065 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:03,070 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:03,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:56:04,722 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:56:07,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:07,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:56:07,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:07,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:07,919 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:08,062 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:56:08,724 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:56:10,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:56:13,064 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:13,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:13,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:13,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:15,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:15,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:56:15,474 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:15,480 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:15,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:56:16,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:56:18,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:23,518 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:24,799 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:24,800 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:56:24,800 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:24,805 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:25,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:56:26,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:56:28,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:28,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:28,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:29,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:33,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:33,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:56:33,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:33,081 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:33,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:56:34,747 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:56:35,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:38,064 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:56:41,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:41,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:41,471 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:56:41,471 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:41,476 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:41,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:56:42,754 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:56:43,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:43,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:43,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:46,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:47,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:47,698 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:56:47,699 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:47,703 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:47,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:56:48,760 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:56:51,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:56,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:56,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:56:56,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:56,016 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:56,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:56:57,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:58,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:58,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:58,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:58,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:01,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:01,367 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:01,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:01,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:01,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:02,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:02,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:05,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:05,485 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:05,485 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:05,490 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:05,775 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:06,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:07,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:08,070 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:57:10,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:10,858 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:10,858 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:10,863 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:11,780 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:12,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:12,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:13,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:13,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:13,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:18,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:18,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:18,378 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:18,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:18,384 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:18,787 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:20,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:23,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:23,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:23,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:23,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:23,526 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:23,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:24,796 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:28,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:28,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:28,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:29,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:30,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:30,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:30,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:30,155 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:30,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:32,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:35,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:35,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:35,649 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:35,649 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:35,654 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:35,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:36,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:38,069 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:57:40,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:40,315 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:40,315 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:40,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:40,320 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:40,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:42,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:43,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:43,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:43,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:45,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:45,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:45,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:45,014 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:45,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:46,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:46,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:50,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:50,423 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:50,424 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:50,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:50,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:51,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:52,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:54,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:54,446 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:54,446 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:54,451 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:54,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:56,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:56,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:58,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:58,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:58,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:59,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:59,754 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:59,754 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:59,759 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:59,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:00,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:01,797 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:06,798 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:08,071 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:58:08,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:08,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:08,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:08,170 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:08,838 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:10,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:12,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:13,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:13,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:13,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:13,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:13,404 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:13,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:13,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:13,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:14,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:17,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:17,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:17,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:17,549 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:17,553 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:17,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:18,850 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:22,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:23,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:23,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:23,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:23,732 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:23,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:24,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:27,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:28,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:28,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:28,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:28,751 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:28,752 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:28,752 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:28,757 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:28,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:30,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:32,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:37,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:37,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:37,906 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:37,906 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:37,911 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:38,074 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:58:38,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:38,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:42,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:42,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:42,807 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:42,808 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:42,812 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:42,872 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:43,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:43,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:43,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:44,875 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:47,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:47,268 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:47,268 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:47,273 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:47,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:48,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:48,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:52,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:52,372 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:52,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:52,377 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:52,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:53,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:54,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:56,983 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:56,984 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:56,984 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:56,989 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:57,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:58,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:58,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:58,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:58,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:58,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:01,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:01,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:01,474 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:01,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:01,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:02,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:03,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:05,894 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:05,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:05,895 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:05,900 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:05,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:06,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:08,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:59:09,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:11,538 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:11,539 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:11,539 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:11,544 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:11,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:12,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:13,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:13,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:13,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:14,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:18,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:18,560 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:18,560 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:18,565 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:18,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:19,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:20,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:24,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:24,053 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:24,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:24,059 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:24,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:25,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:26,919 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:28,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:28,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:28,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:29,232 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:29,233 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:29,233 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:29,237 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:29,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:30,273 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:30,924 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:35,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:35,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:35,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:35,743 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:35,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:35,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:36,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:38,081 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:59:40,932 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:40,933 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:40,933 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:40,934 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:40,938 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:41,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:42,936 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:43,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:43,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:43,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:45,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:45,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:45,498 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:45,503 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:45,938 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:46,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:46,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:51,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:51,318 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:51,318 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:51,322 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:51,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:52,323 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:52,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:56,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:56,004 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:56,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:56,009 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:56,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:56,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:58,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:58,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:58,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:58,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:00,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:00,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:00,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:00,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:00,954 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:02,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:03,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:05,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:05,647 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:05,647 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:05,652 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:05,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:06,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:08,080 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:00:09,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:11,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:11,446 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:11,446 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:11,451 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:11,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:12,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:13,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:13,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:13,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:14,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:16,776 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:16,777 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:16,777 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:16,782 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:16,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:18,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:19,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:21,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:21,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:21,894 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:21,899 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:21,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:22,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:24,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:26,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:26,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:26,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:26,333 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:26,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:28,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:28,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:28,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:28,985 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:30,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:30,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:30,586 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:30,586 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:30,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:30,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:32,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:34,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:34,901 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:34,902 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:34,906 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:34,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:35,907 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:36,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:38,085 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:00:39,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:39,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:39,501 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:39,506 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:39,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:40,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:41,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:43,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:43,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:43,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:45,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:45,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:45,414 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:45,419 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:46,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:47,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:47,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:50,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:50,467 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:50,467 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:50,472 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:51,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:52,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:53,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:57,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:57,249 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:57,249 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:57,254 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:58,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:58,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:58,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:58,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:58,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:59,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:03,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:03,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:03,645 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:03,645 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:03,650 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:04,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:05,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:08,084 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:01:09,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:10,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:10,468 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:10,468 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:10,473 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:11,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:13,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:13,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:13,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:13,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:14,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:16,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:16,874 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:16,875 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:16,880 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:17,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:19,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:19,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:22,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:22,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:22,643 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:22,648 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:23,042 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:25,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:25,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:27,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:27,731 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:27,731 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:27,736 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:28,047 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:28,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:28,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:28,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:29,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:31,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:32,828 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:32,829 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:32,829 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:32,834 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:33,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:35,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:36,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:38,086 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:01:38,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:38,341 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:38,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:38,347 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:39,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:41,061 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:42,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:43,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:43,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:43,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:43,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:43,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:43,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:43,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:44,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:45,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:47,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:50,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:50,341 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:50,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:50,346 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:51,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:53,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:53,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:58,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:58,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:58,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:58,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:58,510 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:58,510 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:58,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:58,515 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:59,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:01,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:03,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:03,228 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:03,228 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:03,233 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:04,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:04,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:05,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:07,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:07,996 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:07,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:08,001 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:08,089 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:02:08,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:09,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:10,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:12,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:12,332 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:12,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:12,338 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:13,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:13,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:13,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:13,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:15,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:15,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:18,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:18,135 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:18,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:18,141 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:19,105 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:19,105 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:21,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:22,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:22,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:22,466 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:22,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:23,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:25,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:26,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:27,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:27,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:27,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:27,025 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:27,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:28,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:28,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:28,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:29,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:31,826 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:31,827 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:31,827 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:31,827 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:31,832 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:32,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:33,120 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:36,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:37,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:37,258 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:37,258 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:37,263 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:38,091 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:02:38,123 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:39,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:42,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:42,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:42,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:42,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:42,155 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:43,131 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:43,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:43,131 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:43,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:43,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:46,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:46,999 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:46,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:47,004 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:47,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:48,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:49,137 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:51,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:51,339 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:51,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:51,346 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:52,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:53,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:53,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:56,072 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:56,073 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:56,073 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:56,079 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:56,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:57,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:58,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:58,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:58,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:59,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:01,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:01,411 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:01,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:01,417 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:02,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:03,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:04,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:07,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:07,485 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:07,485 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:07,490 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:08,097 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:03:08,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:09,159 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:10,098 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:13,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:13,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:13,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:15,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:16,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:16,556 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:16,556 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:16,561 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:17,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:19,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:20,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:24,920 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:24,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:24,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:24,927 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:25,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:25,927 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:27,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:28,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:28,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:28,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:29,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:29,170 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:29,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:29,175 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:29,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:31,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:31,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:35,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:35,310 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:35,311 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:35,316 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:36,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:36,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:37,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:38,095 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:03:40,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:40,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:40,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:40,292 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:41,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:41,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:42,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:43,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:43,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:43,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:45,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:45,234 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:45,234 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:45,239 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:46,192 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:47,195 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:48,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:49,915 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:49,916 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:49,917 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:49,922 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:50,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:51,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:53,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:54,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:54,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:54,202 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:54,207 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:55,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:55,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:58,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:58,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:58,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:59,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:59,350 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:59,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:59,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:59,356 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:00,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:01,211 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:04,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:08,097 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:04:09,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:09,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:09,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:09,918 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:09,923 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:10,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:11,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:13,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:13,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:13,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:15,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:18,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:18,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:18,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:18,527 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:19,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:20,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:21,231 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:25,562 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:26,714 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:26,715 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:26,715 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:26,721 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:27,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:28,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:28,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:28,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:29,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:30,811 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:30,811 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:30,812 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:30,812 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:30,817 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:31,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:33,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:35,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:36,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:36,977 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:36,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:36,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:37,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:38,099 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:04:39,250 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:41,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:41,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:41,714 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:41,714 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:41,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:42,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:43,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:43,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:43,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:43,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:45,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:45,847 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:45,847 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:45,852 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:46,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:46,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:47,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:50,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:50,243 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:50,243 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:50,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:50,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:51,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:52,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:54,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:54,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:54,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:54,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:55,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:57,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:57,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:58,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:58,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:58,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:58,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:58,660 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:58,661 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:58,666 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:59,269 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:01,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:02,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:04,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:04,389 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:04,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:04,394 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:05,279 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:05,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:08,102 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:05:08,103 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:10,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:10,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:10,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:10,779 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:11,283 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:13,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:13,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:13,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:13,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:13,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:17,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:17,999 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:18,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:18,005 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:18,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:19,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:19,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:22,443 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:22,443 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:22,444 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:22,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:23,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:24,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:25,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:26,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:27,000 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:27,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:27,008 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:27,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:28,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:28,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:28,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:29,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:30,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:33,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:33,717 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:33,717 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:33,724 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:34,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:35,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:35,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:37,980 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:37,981 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:37,981 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:37,988 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:38,109 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:05:38,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:39,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:41,111 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:42,976 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:42,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:42,977 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:42,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:43,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:43,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:43,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:43,333 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:45,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:46,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:47,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:47,630 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:47,631 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:47,640 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:48,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:49,349 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:51,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:52,919 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:52,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:52,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:52,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:53,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:55,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:56,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:58,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:58,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:58,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:59,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:59,382 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:59,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:59,390 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:00,359 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:01,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:02,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:04,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:04,219 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:04,219 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:04,229 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:04,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:05,376 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:08,106 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:06:08,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:08,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:08,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:08,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:08,779 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:09,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:11,386 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:13,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:13,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:13,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:13,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:14,931 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:14,932 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:14,932 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:14,941 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:15,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:17,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:18,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:23,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:23,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:23,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:23,401 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:23,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:24,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:25,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:28,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:28,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:28,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:30,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:31,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:31,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:31,612 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:31,623 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:32,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:33,415 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:35,654 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:36,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:36,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:36,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:36,484 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:37,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:37,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:38,111 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:06:40,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:40,871 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:40,872 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:40,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:40,880 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:41,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:43,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:43,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:43,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:43,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:45,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:45,126 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:45,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:45,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:45,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:46,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:47,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:50,263 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:50,263 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:50,264 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:50,271 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:50,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:51,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:51,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:56,272 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:58,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:58,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:58,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:59,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:59,656 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:59,656 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:59,665 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:00,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:01,450 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:01,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:03,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:03,552 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:03,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:03,561 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:04,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:05,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:07,562 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:08,110 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:07:08,256 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:08,257 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:08,257 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:08,265 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:08,462 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:09,467 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:12,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:12,922 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:12,922 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:12,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:12,930 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:13,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:13,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:13,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:13,469 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:15,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:17,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:17,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:17,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:17,575 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:18,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:18,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:19,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:22,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:22,309 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:22,309 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:22,317 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:22,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:23,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:24,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:26,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:26,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:26,674 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:26,680 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:27,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:28,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:28,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:28,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:29,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:29,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:33,263 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:33,264 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:33,264 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:33,271 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:33,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:35,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:35,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:38,112 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:07:41,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:41,676 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:41,677 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:41,678 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:41,683 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:42,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:43,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:43,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:43,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:43,508 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:45,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:45,962 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:45,963 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:45,968 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:46,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:47,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:47,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:52,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:52,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:52,078 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:52,078 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:52,085 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:52,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:53,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:56,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:56,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:56,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:56,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:57,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:57,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:57,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:58,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:58,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:58,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:01,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:01,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:01,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:01,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:02,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:02,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:03,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:07,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:08,121 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:08:11,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:11,138 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:11,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:11,144 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:11,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:13,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:13,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:13,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:13,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:13,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:16,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:16,040 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:16,040 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:16,046 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:16,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:17,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:19,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:21,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:21,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:21,550 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:21,556 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:22,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:23,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:24,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:25,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:25,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:25,820 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:25,825 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:26,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:27,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:28,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:28,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:28,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:30,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:30,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:30,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:30,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:30,819 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:31,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:33,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:35,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:36,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:36,997 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:36,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:37,002 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:37,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:38,117 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:08:39,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:41,118 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:42,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:42,466 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:42,467 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:42,472 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:42,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:43,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:43,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:43,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:43,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:46,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:47,987 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:47,987 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:47,988 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:47,993 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:48,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:49,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:52,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:52,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:52,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:52,475 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:52,480 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:52,581 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:53,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:57,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:57,855 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:57,856 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:57,856 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:57,861 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:58,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:58,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:58,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:58,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:59,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:02,071 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:02,072 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:02,072 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:02,077 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:02,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:03,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:03,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:06,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:06,794 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:06,794 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:06,799 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:07,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:08,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:09:08,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:09,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:12,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:12,950 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:12,950 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:12,956 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:13,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:13,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:13,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:13,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:13,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:15,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:17,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:17,464 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:17,464 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:17,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:17,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:18,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:19,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:22,920 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:22,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:22,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:22,927 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:23,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:23,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:25,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:28,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:28,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:28,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:28,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:28,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:28,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:28,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:29,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:29,624 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:33,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:33,378 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:33,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:33,384 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:33,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:34,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:35,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:38,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:09:40,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:41,707 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:41,708 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:41,708 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:41,713 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:42,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:43,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:43,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:43,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:43,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:45,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:47,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:47,686 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:47,686 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:47,691 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:48,640 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:49,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:50,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:52,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:52,101 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:52,101 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:52,106 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:52,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:53,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:56,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:57,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:57,353 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:57,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:57,359 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:57,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:58,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:58,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:58,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:59,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:01,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:02,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:02,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:02,665 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:02,670 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:03,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:03,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:06,706 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:07,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:07,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:07,612 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:07,617 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:07,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:08,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:10:09,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:12,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:13,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:13,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:13,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:13,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:13,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:13,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:13,346 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:13,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:15,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:17,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:18,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:18,281 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:18,282 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:18,287 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:18,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:19,675 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:23,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:23,091 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:23,091 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:23,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:23,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:23,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:25,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:27,984 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:27,985 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:27,985 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:27,990 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:28,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:28,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:28,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:28,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:28,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:29,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:32,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:32,219 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:32,219 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:32,224 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:32,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:33,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:34,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:36,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:36,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:36,289 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:36,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:36,690 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:37,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:38,126 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:10:40,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:42,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:42,620 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:42,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:42,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:42,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:43,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:43,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:43,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:43,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:45,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:46,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:46,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:46,872 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:46,877 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:47,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:49,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:50,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:54,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:54,792 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:54,792 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:54,797 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:55,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:55,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:56,797 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:58,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:58,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:58,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:59,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:59,456 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:59,456 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:59,462 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:59,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:01,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:02,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:04,562 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:04,563 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:04,563 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:04,568 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:04,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:05,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:07,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:08,128 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:11:10,068 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:10,069 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:10,069 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:10,074 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:10,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:11,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:13,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:13,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:13,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:13,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:16,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:16,557 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:16,557 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:16,562 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:16,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:17,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:18,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:21,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:21,601 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:21,601 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:21,606 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:21,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:23,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:23,741 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:26,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:26,160 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:26,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:26,166 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:26,743 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:27,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:28,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:28,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:28,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:29,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:31,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:31,309 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:31,309 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:31,315 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:31,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:33,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:35,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:38,134 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:11:39,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:39,714 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:39,714 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:39,719 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:39,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:40,720 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:41,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:43,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:43,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:43,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:44,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:44,838 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:44,838 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:44,844 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:45,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:45,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:47,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:50,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:50,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:50,930 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:50,930 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:50,936 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:51,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:53,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:55,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:56,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:56,722 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:56,722 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:56,728 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:56,771 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:57,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:58,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:58,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:58,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:01,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:03,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:03,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:03,554 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:03,559 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:03,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:05,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:06,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:08,132 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:12:11,958 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:11,958 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:11,959 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:11,959 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:11,964 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:12,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:13,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:13,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:13,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:13,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:16,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:16,362 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:16,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:16,367 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:16,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:17,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:17,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:22,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:22,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:22,526 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:22,526 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:22,531 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:22,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:23,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:27,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:27,052 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:27,052 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:27,058 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:27,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:28,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:28,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:28,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:28,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:29,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:31,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:31,355 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:31,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:31,360 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:31,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:33,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:33,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:36,782 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:36,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:36,784 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:36,789 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:36,811 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:37,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:38,137 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:12:39,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:41,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:41,595 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:41,595 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:41,600 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:41,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:43,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:43,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:43,135 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:43,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:44,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:46,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:46,462 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:46,462 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:46,467 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:46,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:47,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:49,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:53,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:53,571 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:53,571 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:53,576 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:53,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:54,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:55,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:57,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:57,421 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:57,422 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:57,427 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:57,831 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:58,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:58,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:58,135 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:59,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:00,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:01,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:01,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:01,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:01,934 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:02,838 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:03,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:05,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:06,746 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:06,747 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:06,747 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:06,752 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:06,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:07,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:08,137 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:13:11,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:12,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:12,342 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:12,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:12,348 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:12,849 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:13,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:13,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:13,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:13,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:16,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:18,538 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:18,538 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:18,538 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:18,543 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:18,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:19,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:21,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:25,476 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:25,477 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:25,477 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:25,482 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:25,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:27,483 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:27,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:28,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:28,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:28,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:31,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:31,228 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:31,228 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:31,234 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:31,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:33,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:33,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:38,139 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:13:38,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:38,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:38,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:38,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:38,503 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:38,875 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:39,878 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:43,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:43,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:43,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:44,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:44,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:44,424 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:44,424 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:44,429 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:44,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:45,884 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:49,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:49,091 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:49,092 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:49,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:49,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:50,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:51,889 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:53,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:53,485 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:53,485 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:53,490 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:53,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:55,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:55,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:58,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:58,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:58,135 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:58,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:58,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:58,436 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:58,441 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:58,896 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:59,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:01,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:05,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:05,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:05,030 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:05,036 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:05,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:07,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:07,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:08,141 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:14:12,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:13,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:13,017 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:13,017 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:13,022 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:13,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:13,135 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:13,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:13,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:15,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:17,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:18,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:18,306 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:18,306 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:18,311 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:18,917 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:19,919 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:23,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:23,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:23,415 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:23,415 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:23,420 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:23,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:25,925 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:28,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:28,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:28,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:28,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:30,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:30,201 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:30,201 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:30,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:30,928 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:31,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:34,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:35,103 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:35,104 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:35,104 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:35,110 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:35,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:37,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:38,144 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:14:40,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:41,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:41,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:41,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:41,641 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:41,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:43,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:43,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:43,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:43,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:45,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:46,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:46,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:46,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:46,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:46,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:47,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:50,850 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:50,851 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:50,851 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:50,851 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:50,856 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:50,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:51,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:55,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:55,694 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:55,694 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:55,699 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:55,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:56,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:57,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:58,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:58,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:58,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:01,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:01,677 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:01,677 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:01,682 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:01,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:01,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:03,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:06,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:07,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:07,579 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:07,579 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:07,584 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:07,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:08,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:15:09,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:12,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:13,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:13,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:13,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:15,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:15,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:15,650 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:15,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:15,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:17,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:17,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:22,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:22,590 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:22,590 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:22,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:22,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:23,596 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:23,985 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:27,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:27,547 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:27,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:27,553 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:27,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:28,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:28,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:28,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:29,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:29,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:31,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:31,887 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:31,887 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:31,892 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:31,992 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:33,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:34,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:37,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:37,676 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:37,676 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:37,682 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:37,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:38,148 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:15:40,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:40,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:42,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:42,491 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:42,491 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:42,495 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:43,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:43,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:43,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:43,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:44,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:45,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:46,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:46,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:46,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:46,813 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:47,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:48,010 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:50,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:52,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:52,495 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:52,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:52,500 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:53,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:54,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:56,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:57,772 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:57,773 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:57,773 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:57,777 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:58,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:58,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:58,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:58,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:00,022 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:02,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:02,686 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:02,686 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:02,686 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:02,691 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:03,025 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:04,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:07,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:07,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:07,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:07,866 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:07,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:08,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:08,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:16:10,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:12,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:12,495 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:12,496 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:12,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:13,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:13,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:13,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:13,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:13,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:14,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:18,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:19,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:19,678 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:19,678 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:19,683 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:20,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:22,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:23,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:28,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:28,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:28,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:28,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:28,420 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:28,420 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:28,425 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:29,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:29,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:30,053 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:32,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:32,688 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:32,688 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:32,693 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:33,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:34,057 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:34,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:36,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:36,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:36,949 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:36,954 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:37,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:38,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:38,153 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:16:40,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:42,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:42,133 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:42,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:42,138 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:43,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:43,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:43,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:43,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:44,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:45,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:48,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:48,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:48,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:48,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:49,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:50,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:51,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:53,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:53,966 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:53,966 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:53,971 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:54,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:56,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:56,971 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:58,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:58,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:58,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:59,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:59,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:59,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:59,322 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:00,092 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:02,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:02,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:04,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:04,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:04,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:04,183 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:05,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:06,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:08,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:17:08,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:09,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:09,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:09,414 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:09,419 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:10,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:12,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:13,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:13,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:13,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:13,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:13,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:13,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:13,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:13,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:14,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:16,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:18,062 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:18,063 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:18,063 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:18,068 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:18,120 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:19,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:20,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:23,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:23,962 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:23,962 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:23,967 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:24,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:24,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:26,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:28,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:28,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:28,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:28,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:28,413 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:28,413 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:28,418 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:29,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:30,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:30,419 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:32,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:32,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:32,212 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:32,217 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:33,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:34,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:36,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:37,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:37,376 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:37,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:37,381 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:38,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:38,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:17:40,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:41,956 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:41,957 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:41,957 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:41,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:41,962 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:42,155 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:43,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:43,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:43,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:44,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:46,256 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:46,257 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:46,257 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:46,262 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:47,162 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:47,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:48,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:50,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:50,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:50,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:50,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:51,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:52,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:52,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:56,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:56,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:56,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:56,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:57,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:58,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:58,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:58,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:58,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:58,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:01,752 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:01,752 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:01,753 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:01,757 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:02,185 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:03,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:04,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:07,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:07,583 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:07,583 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:07,588 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:08,159 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:18:08,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:09,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:10,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:13,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:13,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:13,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:14,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:15,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:15,302 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:15,302 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:15,307 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:16,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:18,211 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:19,916 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:19,917 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:19,917 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:19,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:19,922 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:20,213 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:22,217 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:24,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:25,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:25,431 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:25,431 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:25,436 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:26,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:28,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:28,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:28,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:28,226 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:29,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:29,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:29,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:29,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:30,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:30,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:32,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:33,984 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:33,984 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:33,985 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:33,990 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:34,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:35,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:36,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:38,165 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:18:38,851 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:38,852 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:38,852 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:38,857 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:39,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:40,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:41,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:43,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:43,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:43,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:43,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:43,384 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:43,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:43,389 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:44,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:46,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:47,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:48,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:48,036 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:48,036 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:48,042 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:48,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:50,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:52,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:52,803 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:52,803 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:52,803 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:52,808 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:53,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:54,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:57,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:57,205 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:57,206 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:57,211 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:57,267 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:58,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:58,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:58,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:58,269 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:58,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:01,735 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:01,736 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:01,736 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:01,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:02,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:03,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:04,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:07,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:07,229 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:07,229 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:07,234 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:07,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:08,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:19:08,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:09,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:13,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:13,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:13,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:13,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:13,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:13,534 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:13,540 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:14,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:14,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:16,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:19,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:19,034 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:19,035 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:19,040 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:19,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:20,040 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:20,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:22,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:22,871 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:22,872 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:22,876 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:23,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:24,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:25,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:27,491 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:27,492 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:27,492 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:27,497 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:28,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:28,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:28,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:28,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:30,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:31,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:32,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:32,190 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:32,190 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:32,195 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:32,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:34,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:36,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:36,850 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:36,851 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:36,851 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:36,855 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:37,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:38,166 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:19:38,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:41,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:41,392 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:41,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:41,397 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:42,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:42,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:43,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:43,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:43,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:44,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:48,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:48,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:48,647 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:48,648 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:48,652 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:49,344 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:50,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:53,690 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:54,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:54,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:54,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:54,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:55,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:56,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:58,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:58,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:58,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:59,091 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:59,092 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:59,092 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:59,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:59,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:59,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:00,362 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:04,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:07,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:07,621 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:07,621 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:07,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:08,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:20:08,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:09,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:10,376 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:12,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:12,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:12,288 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:12,293 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:12,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:13,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:13,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:13,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:14,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:14,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:17,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:17,107 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:17,107 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:17,112 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:17,385 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:18,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:20,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:22,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:22,506 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:22,507 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:22,512 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:23,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:24,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:25,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:28,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:28,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:28,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:28,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:28,365 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:28,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:28,371 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:28,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:30,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:31,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:32,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:32,610 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:32,611 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:32,615 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:33,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:34,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:36,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:38,170 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:20:38,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:38,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:38,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:38,553 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:39,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:40,420 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:42,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:43,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:43,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:43,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:44,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:44,594 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:44,594 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:44,600 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:45,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:46,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:47,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:49,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:49,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:49,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:49,318 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:49,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:50,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:53,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:54,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:54,994 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:54,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:54,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:55,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:56,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:58,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:58,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:58,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:58,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:03,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:03,192 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:03,193 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:03,197 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:03,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:04,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:04,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:08,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:21:08,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:08,971 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:08,971 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:08,976 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:09,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:09,976 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:10,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:13,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:13,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:13,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:14,656 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:14,657 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:14,658 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:14,664 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:15,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:15,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:16,474 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:20,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:21,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:21,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:21,227 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:21,232 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:21,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:22,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:25,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:25,549 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:25,549 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:25,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:26,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:26,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:28,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:28,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:28,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:28,491 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:30,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:30,780 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:30,781 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:30,785 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:31,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:31,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:32,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:34,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:34,996 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:34,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:35,001 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:35,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:36,503 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:37,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:38,175 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:21:42,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:42,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:42,800 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:42,801 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:42,805 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:43,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:43,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:43,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:43,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:44,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:47,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:49,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:49,347 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:49,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:49,351 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:49,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:50,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:52,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:53,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:53,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:53,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:53,845 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:54,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:56,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:57,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:58,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:58,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:58,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:58,099 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:58,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:58,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:58,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:58,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:00,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:03,424 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:04,659 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:04,660 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:04,660 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:04,665 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:05,533 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:06,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:08,180 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:22:09,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:09,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:09,846 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:09,846 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:09,851 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:10,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:12,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:13,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:13,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:13,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:14,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:15,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:15,254 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:15,254 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:15,259 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:15,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:16,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:20,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:21,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:21,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:21,058 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:21,064 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:21,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:22,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:25,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:25,939 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:25,939 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:25,939 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:25,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:26,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:28,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:28,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:28,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:28,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:31,292 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:31,292 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:31,293 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:31,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:31,298 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:31,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:32,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:36,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:36,218 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:36,219 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:36,223 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:36,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:37,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:38,179 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:22:38,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:43,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:43,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:43,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:43,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:44,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:44,765 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:44,765 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:44,770 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:45,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:46,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:48,805 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:49,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:49,074 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:49,074 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:49,079 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:49,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:50,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:54,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:56,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:56,627 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:56,628 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:56,633 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:57,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:58,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:58,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:58,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:58,588 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:59,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:01,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:01,005 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:01,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:01,010 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:01,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:02,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:05,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:05,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:05,726 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:05,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:05,732 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:06,597 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:08,181 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:23:08,600 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:10,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:11,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:11,067 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:11,067 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:11,072 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:11,603 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:12,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:13,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:13,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:13,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:15,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:15,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:15,542 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:15,542 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:15,548 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:15,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:16,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:20,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:22,479 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:22,480 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:22,480 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:22,486 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:22,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:24,624 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:26,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:26,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:26,993 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:26,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:27,001 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:27,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:28,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:28,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:28,633 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:31,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:31,513 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:31,513 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:31,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:31,524 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:31,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:32,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:36,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:36,147 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:36,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:36,153 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:36,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:37,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:38,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:23:38,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:41,040 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:41,041 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:41,041 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:41,050 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:41,650 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:42,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:43,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:43,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:43,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:43,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:48,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:48,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:48,227 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:48,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:48,234 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:48,655 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:50,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:52,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:52,118 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:52,118 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:52,125 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:52,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:54,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:54,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:57,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:57,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:57,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:57,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:57,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:58,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:58,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:58,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:58,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:59,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:04,546 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:06,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:06,105 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:06,106 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:06,111 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:06,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:08,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:24:08,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:10,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:10,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:10,613 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:10,613 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:10,618 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:10,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:12,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:13,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:13,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:13,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:15,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:16,054 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:16,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:16,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:16,060 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:16,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:18,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:20,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:20,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:20,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:20,915 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:20,922 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:21,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:22,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:25,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:27,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:27,407 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:27,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:27,414 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:27,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:28,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:28,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:28,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:31,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:33,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:33,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:33,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:33,454 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:33,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:34,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:36,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:38,192 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:24:41,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:41,730 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:41,730 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:41,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:41,736 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:42,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:43,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:43,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:43,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:44,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:46,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:46,136 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:46,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:46,141 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:46,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:47,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:48,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:51,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:51,200 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:51,200 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:51,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:51,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:52,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:52,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:56,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:56,251 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:56,251 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:56,256 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:56,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:57,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:58,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:58,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:58,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:58,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:00,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:00,345 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:00,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:00,351 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:00,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:02,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:02,741 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:05,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:05,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:05,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:05,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:06,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:07,860 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:08,190 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:25:08,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:12,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:12,045 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:12,045 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:12,050 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:12,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:13,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:13,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:13,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:13,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:14,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:16,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:16,114 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:16,114 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:16,119 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:16,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:18,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:18,760 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:21,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:21,562 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:21,562 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:21,567 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:21,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:22,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:23,568 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:26,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:26,226 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:26,226 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:26,232 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:26,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:28,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:28,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:29,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:30,176 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:30,177 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:30,177 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:30,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:30,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:32,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:35,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:36,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:36,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:36,047 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:36,053 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:36,776 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:38,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:25:38,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:41,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:43,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:43,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:43,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:43,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:43,363 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:43,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:43,368 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:43,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:44,786 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:46,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:47,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:47,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:47,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:47,439 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:47,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:48,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:51,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:55,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:55,003 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:55,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:55,008 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:55,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:56,796 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:57,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:58,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:58,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:58,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:00,268 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:00,269 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:00,269 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:00,275 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:00,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:02,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:02,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:05,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:05,374 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:05,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:05,379 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:05,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:06,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:07,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:08,194 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:26:10,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:10,133 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:10,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:10,139 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:10,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:12,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:13,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:13,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:13,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:13,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:14,826 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:14,827 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:14,827 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:14,832 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:15,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:16,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:18,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:19,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:19,082 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:19,082 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:19,087 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:19,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:20,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:24,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:24,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:24,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:24,212 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:24,217 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:24,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:26,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:28,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:29,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:29,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:29,706 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:29,706 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:29,711 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:29,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:30,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:34,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:34,254 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:34,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:34,261 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:34,832 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:35,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:36,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:38,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:26:39,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:39,354 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:39,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:39,360 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:39,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:40,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:40,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:43,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:43,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:43,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:43,908 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:43,908 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:43,909 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:43,914 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:44,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:45,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:46,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:49,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:49,251 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:49,251 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:49,256 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:49,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:50,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:51,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:53,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:53,882 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:53,882 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:53,888 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:54,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:56,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:56,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:58,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:58,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:58,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:58,415 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:58,415 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:58,416 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:58,425 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:58,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:00,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:02,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:02,986 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:02,987 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:02,987 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:02,992 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:03,863 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:04,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:07,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:08,199 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:27:10,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:10,541 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:10,541 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:10,546 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:10,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:12,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:13,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:13,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:13,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:13,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:15,544 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:15,544 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:15,545 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:15,550 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:15,875 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:16,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:18,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:20,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:20,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:20,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:20,283 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:20,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:22,884 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:24,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:24,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:24,698 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:24,699 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:24,703 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:24,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:26,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:28,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:29,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:29,244 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:29,244 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:29,249 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:29,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:29,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:30,892 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:34,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:34,805 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:34,806 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:34,807 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:34,812 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:34,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:36,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:38,201 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:27:40,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:43,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:43,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:43,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:43,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:43,364 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:43,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:43,369 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:43,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:44,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:45,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:49,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:49,080 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:49,080 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:49,085 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:49,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:50,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:51,086 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:54,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:54,249 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:54,249 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:54,254 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:54,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:56,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:56,917 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:58,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:58,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:58,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:58,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:58,356 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:58,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:58,361 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:58,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:00,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:01,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:04,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:04,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:04,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:04,177 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:04,925 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:06,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:07,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:08,207 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:28:10,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:10,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:10,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:10,283 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:10,932 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:12,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:12,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:13,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:13,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:13,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:15,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:15,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:15,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:15,779 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:15,936 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:16,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:17,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:21,178 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:21,179 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:21,179 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:21,184 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:21,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:22,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:23,185 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:27,755 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:27,756 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:27,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:27,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:27,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:28,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:28,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:28,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:28,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:33,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:34,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:34,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:34,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:34,740 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:34,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:36,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:38,205 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:28:39,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:39,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:39,383 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:39,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:39,389 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:39,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:40,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:43,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:43,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:43,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:44,299 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:44,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:44,300 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:44,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:44,305 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:44,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:46,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:49,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:49,128 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:49,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:49,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:49,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:50,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:50,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:54,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:54,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:54,483 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:54,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:54,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:55,488 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:56,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:58,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:58,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:58,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:58,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:58,927 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:58,927 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:58,932 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:58,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:00,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:00,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:05,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:06,744 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:06,745 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:06,745 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:06,750 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:06,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:08,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:29:08,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:11,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:11,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:11,554 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:11,554 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:11,559 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:11,992 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:12,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:13,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:13,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:13,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:16,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:18,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:18,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:18,674 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:18,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:18,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:21,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:21,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:23,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:23,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:23,660 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:23,668 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:24,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:25,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:27,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:28,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:28,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:28,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:28,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:28,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:28,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:28,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:29,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:31,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:32,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:32,430 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:32,431 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:32,436 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:33,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:33,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:35,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:36,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:36,893 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:36,893 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:36,898 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:37,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:38,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:29:39,024 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:39,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:42,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:42,939 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:42,939 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:42,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:43,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:43,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:43,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:43,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:44,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:45,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:48,532 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:48,533 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:48,533 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:48,538 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:49,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:49,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:51,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:53,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:53,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:53,038 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:53,043 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:54,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:55,042 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:55,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:58,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:58,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:58,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:59,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:59,092 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:59,093 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:59,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:00,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:00,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:01,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:04,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:04,042 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:04,042 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:04,047 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:04,051 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:05,053 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:06,047 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:08,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:08,146 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:08,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:08,151 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:08,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:30:09,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:11,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:11,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:13,060 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:13,061 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:13,061 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:13,066 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:13,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:13,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:13,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:14,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:15,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:16,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:17,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:17,530 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:17,530 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:17,535 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:18,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:19,069 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:21,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:21,822 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:21,823 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:21,823 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:21,828 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:22,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:23,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:26,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:26,377 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:26,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:26,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:27,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:27,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:28,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:28,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:28,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:29,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:30,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:30,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:30,926 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:30,931 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:31,081 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:32,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:33,084 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:36,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:36,715 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:36,715 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:36,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:37,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:38,214 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:30:38,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:39,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:42,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:42,359 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:42,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:42,364 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:43,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:43,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:43,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:43,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:43,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:45,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:48,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:51,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:51,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:51,878 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:51,883 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:52,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:53,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:53,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:57,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:57,345 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:57,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:57,350 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:58,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:58,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:58,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:58,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:59,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:59,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:02,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:02,614 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:02,614 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:02,619 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:03,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:04,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:05,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:07,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:07,050 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:07,051 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:07,056 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:07,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:08,216 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:31:09,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:10,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:11,536 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:11,537 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:11,537 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:11,542 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:12,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:13,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:13,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:13,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:13,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:15,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:15,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:15,656 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:15,656 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:15,661 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:16,129 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:17,131 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:20,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:20,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:20,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:20,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:20,519 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:21,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:23,137 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:24,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:24,967 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:24,967 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:24,972 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:25,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:25,972 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:27,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:28,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:28,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:28,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:31,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:31,454 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:31,455 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:31,455 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:31,460 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:32,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:33,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:36,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:37,712 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:37,713 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:37,713 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:37,719 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:38,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:38,222 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:31:39,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:42,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:42,062 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:42,062 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:42,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:42,067 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:42,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:43,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:43,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:43,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:43,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:46,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:46,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:46,758 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:46,763 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:47,159 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:47,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:49,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:52,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:53,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:53,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:53,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:53,081 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:53,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:55,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:58,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:58,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:58,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:58,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:03,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:03,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:03,673 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:03,674 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:03,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:04,173 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:05,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:08,191 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:08,192 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:08,192 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:08,197 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:08,220 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:32:09,178 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:09,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:09,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:13,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:13,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:13,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:14,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:16,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:16,418 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:16,419 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:16,423 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:17,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:19,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:19,462 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:22,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:22,997 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:22,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:23,002 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:23,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:25,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:25,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:28,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:28,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:28,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:30,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:31,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:31,424 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:31,424 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:31,429 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:32,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:33,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:35,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:37,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:37,917 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:37,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:37,923 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:38,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:38,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:32:39,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:41,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:42,924 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:42,925 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:42,925 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:42,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:43,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:43,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:43,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:43,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:45,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:46,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:49,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:49,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:49,474 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:49,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:50,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:51,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:51,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:53,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:53,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:53,659 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:53,664 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:54,220 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:55,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:56,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:58,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:58,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:58,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:59,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:59,822 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:59,822 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:59,827 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:00,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:01,226 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:01,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:06,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:06,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:06,664 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:06,669 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:07,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:07,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:08,225 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:33:09,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:13,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:13,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:13,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:13,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:16,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:16,603 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:16,603 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:16,608 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:17,242 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:18,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:19,245 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:21,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:21,795 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:21,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:21,800 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:22,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:23,250 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:23,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:28,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:28,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:28,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:28,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:28,456 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:28,456 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:28,461 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:29,254 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:29,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:31,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:34,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:35,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:35,188 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:35,188 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:35,193 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:35,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:37,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:38,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:33:40,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:40,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:40,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:40,322 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:40,327 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:41,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:43,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:43,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:43,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:43,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:44,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:44,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:44,878 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:44,883 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:45,272 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:45,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:47,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:49,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:49,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:49,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:49,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:50,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:51,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:51,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:55,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:55,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:55,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:55,439 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:56,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:57,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:57,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:58,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:58,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:58,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:00,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:00,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:00,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:00,127 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:00,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:01,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:03,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:05,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:05,090 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:05,090 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:05,095 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:05,292 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:07,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:08,233 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:34:08,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:11,805 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:11,806 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:11,806 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:11,811 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:12,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:13,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:13,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:13,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:13,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:13,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:18,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:19,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:19,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:19,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:19,202 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:19,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:21,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:24,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:24,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:24,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:24,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:24,216 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:24,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:25,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:28,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:28,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:28,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:29,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:30,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:30,045 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:30,045 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:30,050 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:30,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:31,320 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:34,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:34,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:34,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:34,349 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:35,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:35,323 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:35,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:38,232 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:34:38,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:38,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:38,639 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:38,644 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:39,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:40,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:41,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:43,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:43,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:43,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:44,562 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:44,563 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:44,563 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:44,568 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:45,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:46,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:47,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:49,808 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:49,809 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:49,810 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:49,815 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:50,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:51,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:51,815 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:55,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:55,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:55,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:55,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:55,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:57,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:57,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:58,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:58,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:58,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:59,940 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:59,940 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:59,941 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:59,946 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:00,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:01,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:02,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:07,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:07,371 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:07,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:07,376 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:08,237 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:35:08,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:08,353 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:09,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:13,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:13,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:13,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:13,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:15,279 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:15,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:15,280 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:15,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:15,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:17,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:19,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:19,424 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:19,425 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:19,425 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:19,430 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:20,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:21,368 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:24,431 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:26,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:26,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:26,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:26,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:26,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:27,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:28,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:28,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:28,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:30,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:30,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:30,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:30,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:30,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:31,376 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:33,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:35,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:36,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:36,411 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:36,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:36,417 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:37,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:37,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:38,236 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:35:41,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:41,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:41,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:41,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:41,159 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:41,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:43,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:43,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:43,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:43,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:46,188 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:46,189 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:46,189 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:46,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:46,194 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:46,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:47,395 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:51,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:52,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:52,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:52,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:52,214 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:52,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:53,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:57,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:57,216 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:57,216 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:57,216 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:57,221 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:57,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:58,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:58,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:58,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:59,407 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:02,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:02,189 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:02,189 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:02,195 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:02,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:02,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:03,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:07,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:08,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:36:11,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:11,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:11,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:11,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:11,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:13,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:13,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:13,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:13,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:13,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:17,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:17,243 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:17,244 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:17,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:17,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:18,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:19,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:21,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:21,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:21,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:21,739 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:22,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:23,428 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:23,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:27,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:27,066 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:27,067 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:27,072 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:27,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:28,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:28,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:28,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:29,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:29,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:34,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:36,040 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:36,041 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:36,042 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:36,047 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:36,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:37,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:38,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:36:40,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:40,527 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:40,527 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:40,528 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:40,532 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:41,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:43,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:43,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:43,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:43,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:45,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:45,131 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:45,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:45,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:45,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:46,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:47,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:51,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:53,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:53,688 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:53,688 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:53,692 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:54,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:55,458 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:56,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:58,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:58,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:58,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:59,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:59,379 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:59,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:59,383 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:59,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:01,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:02,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:07,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:08,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:08,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:08,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:08,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:08,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:37:08,469 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:09,472 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:12,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:12,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:12,743 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:12,744 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:12,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:13,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:13,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:13,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:13,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:15,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:17,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:17,426 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:17,426 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:17,431 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:17,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:18,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:19,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:22,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:22,184 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:22,184 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:22,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:22,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:23,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:24,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:28,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:28,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:28,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:28,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:28,476 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:28,476 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:28,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:28,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:29,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:29,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:33,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:33,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:33,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:33,133 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:33,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:35,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:35,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:37,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:37,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:37,807 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:37,812 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:38,247 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:37:38,496 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:39,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:40,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:42,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:42,116 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:42,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:42,121 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:42,501 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:43,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:43,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:43,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:43,503 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:45,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:48,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:48,270 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:48,271 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:48,275 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:48,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:49,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:51,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:54,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:54,234 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:54,234 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:54,240 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:54,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:55,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:57,240 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:58,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:58,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:58,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:59,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:59,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:59,674 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:59,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:00,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:01,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:02,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:05,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:05,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:05,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:05,317 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:05,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:07,527 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:08,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:38:08,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:13,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:13,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:13,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:13,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:13,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:13,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:13,664 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:13,669 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:14,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:15,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:17,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:17,887 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:17,887 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:17,892 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:18,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:18,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:19,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:23,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:25,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:25,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:25,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:25,780 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:26,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:27,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:28,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:28,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:28,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:29,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:30,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:30,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:30,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:30,099 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:30,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:31,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:35,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:36,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:36,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:36,223 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:36,228 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:36,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:37,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:38,248 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:38:40,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:41,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:41,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:41,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:41,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:42,560 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:43,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:43,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:43,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:43,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:45,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:46,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:46,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:46,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:46,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:46,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:47,567 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:49,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:49,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:49,949 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:49,954 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:50,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:50,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:51,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:55,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:55,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:55,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:55,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:55,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:56,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:57,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:58,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:58,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:58,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:01,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:03,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:03,513 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:03,513 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:03,519 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:03,581 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:05,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:06,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:08,254 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:39:08,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:08,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:08,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:08,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:08,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:09,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:11,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:13,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:13,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:13,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:13,244 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:13,503 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:13,503 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:13,507 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:13,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:15,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:17,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:17,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:17,939 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:17,939 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:17,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:18,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:19,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:22,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:24,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:24,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:24,930 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:24,935 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:25,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:27,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:28,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:28,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:28,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:28,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:30,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:30,490 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:30,490 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:30,495 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:30,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:31,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:33,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:36,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:36,961 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:36,961 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:36,966 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:37,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:38,253 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:39:39,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:39,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:43,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:43,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:43,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:43,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:43,513 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:43,513 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:43,518 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:43,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:44,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:45,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:49,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:49,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:49,731 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:49,731 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:49,736 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:50,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:51,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:54,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:54,866 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:54,867 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:54,868 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:54,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:55,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:57,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:58,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:58,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:58,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:59,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:59,206 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:59,206 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:59,211 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:59,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:00,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:01,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:04,574 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:04,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:04,575 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:04,580 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:04,645 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:05,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:05,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:08,258 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:40:11,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:11,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:11,795 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:11,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:11,800 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:12,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:13,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:13,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:13,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:13,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:16,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:18,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:18,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:18,312 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:18,317 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:18,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:19,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:22,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:24,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:24,645 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:24,646 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:24,651 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:24,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:25,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:27,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:28,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:28,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:28,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:31,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:31,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:31,487 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:31,492 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:31,669 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:33,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:33,672 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:36,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:36,141 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:36,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:36,146 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:36,674 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:37,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:38,258 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:40:39,259 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:42,070 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:42,071 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:42,071 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:42,076 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:42,680 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:43,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:43,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:43,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:43,682 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:44,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:49,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:50,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:50,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:50,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:50,526 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:50,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:51,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:54,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:56,840 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:56,841 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:56,841 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:56,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:57,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:58,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:58,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:58,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:59,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:00,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:05,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:05,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:05,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:05,501 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:05,507 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:05,701 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:41:07,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:08,260 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:41:11,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:13,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:13,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:13,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:16,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:16,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:16,943 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:16,943 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:16,948 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:17,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:41:19,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:21,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:23,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:23,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:23,550 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:23,555 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:23,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:41:25,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:27,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:28,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:28,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:28,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:30,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:30,738 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:30,738 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:30,743 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:31,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:41:31,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:32,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:35,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:35,203 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:35,203 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:35,208 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:35,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:41:37,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:38,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:38,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:41:42,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:42,971 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:42,971 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:42,976 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:43,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:43,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:43,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:43,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:43,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:41:45,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:47,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:47,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:47,442 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:47,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:47,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:41:48,485 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:49,743 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:51,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:51,939 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:51,939 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:51,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:52,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:41:53,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:53,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:58,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:58,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:58,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:59,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:59,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:59,941 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:59,942 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:59,946 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:00,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:01,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:04,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:04,558 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:04,558 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:04,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:04,563 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:04,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:07,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:08,264 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:42:09,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:09,456 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:09,456 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:09,461 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:09,763 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:10,462 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:11,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:13,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:13,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:13,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:15,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:15,384 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:15,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:15,389 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:15,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:16,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:17,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:20,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:20,139 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:20,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:20,144 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:20,776 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:21,778 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:22,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:26,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:26,625 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:26,625 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:26,630 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:26,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:27,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:28,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:28,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:28,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:29,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:31,096 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:31,097 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:31,097 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:31,102 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:31,786 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:33,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:33,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:36,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:36,131 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:36,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:36,137 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:36,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:37,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:38,270 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:42:38,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:41,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:41,198 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:41,198 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:41,203 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:41,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:43,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:43,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:43,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:43,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:43,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:46,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:46,637 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:46,637 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:46,642 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:46,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:48,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:49,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:51,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:51,631 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:51,632 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:51,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:51,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:53,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:54,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:58,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:58,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:58,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:59,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:59,258 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:59,259 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:59,264 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:59,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:00,264 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:01,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:04,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:04,346 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:04,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:04,353 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:04,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:05,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:05,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:08,269 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:43:10,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:10,834 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:10,835 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:10,835 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:10,839 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:11,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:13,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:13,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:13,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:13,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:15,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:15,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:15,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:15,348 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:15,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:16,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:17,831 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:21,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:21,138 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:21,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:21,143 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:21,834 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:22,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:23,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:27,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:28,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:28,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:28,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:28,296 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:28,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:28,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:28,354 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:28,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:29,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:32,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:32,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:32,568 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:32,568 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:32,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:32,844 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:35,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:37,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:38,274 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:43:39,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:39,565 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:39,565 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:39,570 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:39,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:41,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:43,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:43,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:43,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:43,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:44,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:44,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:44,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:44,559 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:44,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:47,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:48,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:49,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:49,107 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:49,107 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:49,112 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:49,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:51,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:53,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:53,569 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:53,570 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:53,575 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:53,867 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:54,575 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:55,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:57,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:57,711 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:57,711 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:57,716 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:57,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:58,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:58,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:58,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:59,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:00,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:02,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:02,615 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:02,615 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:02,620 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:02,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:05,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:05,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:07,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:07,456 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:07,457 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:07,462 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:07,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:08,273 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:44:09,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:11,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:12,415 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:12,416 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:12,417 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:12,422 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:12,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:13,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:13,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:13,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:13,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:16,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:16,987 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:16,987 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:16,988 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:16,993 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:17,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:19,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:21,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:21,974 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:21,974 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:21,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:21,979 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:22,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:23,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:26,980 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:28,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:28,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:28,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:30,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:30,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:30,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:30,353 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:30,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:31,908 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:32,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:34,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:34,842 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:34,843 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:34,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:34,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:37,848 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:37,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:38,275 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:44:40,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:40,611 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:40,611 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:40,616 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:40,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:43,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:43,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:43,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:43,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:43,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:47,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:47,240 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:47,240 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:47,244 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:47,924 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:49,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:49,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:52,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:52,315 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:52,316 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:52,320 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:52,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:53,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:54,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:57,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:57,018 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:57,018 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:57,023 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:57,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:58,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:58,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:58,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:59,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:59,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:01,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:01,712 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:01,713 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:01,717 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:01,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:03,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:04,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:07,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:07,125 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:07,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:07,130 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:07,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:08,281 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:45:09,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:10,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:11,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:11,993 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:11,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:11,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:12,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:13,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:13,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:13,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:13,953 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:15,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:16,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:16,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:16,482 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:16,487 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:16,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:17,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:20,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:24,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:24,644 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:24,644 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:24,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:24,962 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:25,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:27,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:28,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:28,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:28,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:30,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:30,838 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:30,838 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:30,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:30,843 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:30,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:33,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:35,736 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:35,737 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:35,737 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:35,742 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:35,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:36,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:37,976 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:38,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:45:42,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:43,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:43,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:43,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:43,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:43,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:43,787 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:43,792 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:43,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:45,985 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:47,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:48,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:48,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:48,866 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:48,871 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:48,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:51,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:52,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:57,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:57,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:57,323 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:57,328 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:57,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:58,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:58,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:58,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:58,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:59,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:02,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:02,946 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:02,947 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:02,952 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:02,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:03,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:06,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:08,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:46:08,972 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:08,973 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:08,973 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:08,979 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:09,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:09,979 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:12,010 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:13,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:13,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:13,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:13,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:13,424 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:13,424 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:13,429 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:14,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:15,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:16,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:19,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:19,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:19,322 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:19,327 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:20,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:21,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:22,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:24,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:24,889 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:24,889 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:24,895 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:25,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:26,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:28,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:28,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:28,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:28,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:30,511 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:30,512 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:30,512 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:30,517 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:31,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:32,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:32,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:35,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:35,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:35,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:35,739 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:36,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:37,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:38,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:38,284 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:46:41,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:41,193 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:41,193 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:41,198 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:42,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:43,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:43,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:43,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:43,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:44,040 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:46,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:46,381 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:46,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:46,386 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:47,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:48,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:49,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:52,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:52,443 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:52,443 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:52,448 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:53,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:54,051 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:54,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:56,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:56,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:56,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:56,949 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:57,053 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:58,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:58,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:58,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:00,057 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:00,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:01,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:01,167 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:01,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:01,173 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:02,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:04,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:05,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:05,401 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:05,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:05,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:05,406 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:06,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:08,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:08,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:47:11,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:13,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:13,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:13,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:14,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:14,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:14,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:14,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:15,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:16,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:16,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:19,783 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:19,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:19,784 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:19,789 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:20,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:21,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:22,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:24,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:24,313 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:24,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:24,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:25,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:26,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:27,319 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:28,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:28,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:28,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:30,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:30,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:30,213 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:30,218 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:31,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:32,096 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:33,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:35,986 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:35,987 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:35,988 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:35,993 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:36,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:38,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:38,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:47:38,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:40,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:40,503 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:40,503 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:40,508 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:41,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:42,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:43,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:43,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:43,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:43,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:45,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:45,479 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:45,479 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:45,484 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:46,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:48,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:48,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:51,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:51,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:51,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:51,446 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:52,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:54,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:54,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:58,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:58,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:58,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:00,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:00,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:00,821 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:00,822 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:00,826 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:01,123 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:04,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:05,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:05,549 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:05,549 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:05,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:05,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:06,129 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:08,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:08,291 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:48:10,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:10,289 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:10,289 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:10,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:11,135 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:11,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:12,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:13,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:13,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:13,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:14,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:14,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:14,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:14,919 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:15,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:16,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:18,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:21,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:21,700 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:21,701 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:21,706 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:22,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:22,706 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:24,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:26,518 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:26,519 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:26,519 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:26,524 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:27,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:28,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:28,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:28,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:28,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:28,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:31,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:31,437 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:31,437 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:31,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:32,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:33,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:34,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:36,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:36,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:36,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:36,312 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:37,162 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:38,165 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:38,296 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:48:39,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:42,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:42,404 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:42,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:42,409 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:43,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:43,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:44,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:44,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:47,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:47,215 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:47,215 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:47,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:48,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:50,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:50,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:54,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:54,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:54,501 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:54,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:55,178 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:55,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:56,178 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:58,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:58,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:58,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:59,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:59,336 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:59,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:59,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:00,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:01,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:02,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:03,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:03,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:03,898 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:03,903 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:04,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:06,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:06,903 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:08,295 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:49:08,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:08,677 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:08,677 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:08,682 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:09,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:10,196 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:12,683 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:13,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:13,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:13,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:16,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:16,114 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:16,115 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:16,120 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:16,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:18,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:18,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:21,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:21,285 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:21,285 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:21,290 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:22,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:23,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:24,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:26,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:26,366 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:26,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:26,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:27,209 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:28,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:28,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:28,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:28,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:28,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:31,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:31,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:31,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:31,243 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:32,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:34,217 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:34,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:35,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:35,326 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:35,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:35,331 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:36,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:38,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:38,300 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:49:39,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:40,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:40,635 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:40,635 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:40,640 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:41,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:42,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:43,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:43,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:43,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:44,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:46,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:46,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:46,778 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:46,783 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:47,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:48,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:49,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:51,246 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:51,247 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:51,247 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:51,252 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:52,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:54,238 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:55,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:57,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:57,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:57,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:57,188 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:57,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:58,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:58,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:58,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:00,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:00,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:02,272 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:02,273 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:02,273 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:02,278 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:03,246 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:04,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:06,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:08,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:50:10,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:10,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:10,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:10,303 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:11,256 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:12,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:12,304 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:13,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:13,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:13,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:15,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:15,667 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:15,667 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:15,672 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:16,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:17,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:18,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:22,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:22,511 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:22,511 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:22,516 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:23,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:23,516 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:24,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:26,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:26,738 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:26,739 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:26,743 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:27,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:28,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:28,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:28,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:28,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:29,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:30,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:30,968 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:30,968 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:30,973 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:31,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:34,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:35,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:35,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:35,347 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:35,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:35,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:36,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:38,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:38,301 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:50:40,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:43,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:43,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:43,785 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:43,785 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:43,790 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:44,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:45,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:46,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:49,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:49,665 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:49,666 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:49,670 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:50,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:51,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:52,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:56,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:58,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:58,111 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:58,111 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:58,117 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:58,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:58,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:58,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:58,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:00,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:02,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:02,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:02,614 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:02,614 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:02,620 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:03,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:04,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:07,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:08,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:51:08,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:08,325 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:08,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:08,331 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:09,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:10,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:13,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:13,078 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:13,078 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:13,078 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:13,083 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:13,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:13,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:13,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:13,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:16,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:18,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:18,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:18,620 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:18,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:18,625 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:19,321 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:20,323 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:23,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:23,725 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:23,726 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:23,726 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:23,732 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:24,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:26,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:27,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:27,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:27,969 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:27,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:28,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:28,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:28,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:28,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:29,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:30,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:33,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:33,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:33,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:33,014 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:33,336 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:35,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:36,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:38,306 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:51:40,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:40,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:40,044 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:40,049 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:40,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:41,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:42,346 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:43,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:44,884 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:44,885 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:44,885 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:44,890 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:45,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:46,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:46,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:51,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:51,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:51,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:51,889 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:52,355 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:52,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:54,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:57,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:58,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:58,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:58,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:00,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:00,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:00,223 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:00,228 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:00,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:02,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:03,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:08,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:08,311 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:52:13,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:13,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:13,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:13,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:16,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:16,123 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:16,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:16,129 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:16,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:18,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:19,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:21,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:21,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:21,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:21,135 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:21,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:24,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:25,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:25,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:25,781 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:25,782 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:25,786 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:26,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:28,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:28,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:28,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:28,388 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:30,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:31,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:31,490 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:31,490 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:31,495 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:32,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:34,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:35,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:38,310 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:52:39,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:39,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:39,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:39,292 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:39,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:41,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:42,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:43,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:44,431 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:44,431 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:44,432 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:44,437 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:45,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:46,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:46,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:49,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:49,379 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:49,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:49,385 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:49,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:52,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:52,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:53,715 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:53,716 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:53,716 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:53,721 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:54,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:56,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:57,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:58,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:58,014 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:58,014 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:58,019 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:58,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:58,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:58,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:58,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:00,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:02,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:02,595 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:02,596 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:02,601 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:03,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:03,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:04,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:08,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:08,083 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:08,083 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:08,088 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:08,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:53:08,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:09,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:10,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:13,118 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:13,118 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:13,119 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:13,123 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:13,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:13,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:13,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:13,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:14,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:14,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:17,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:17,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:17,865 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:17,870 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:18,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:19,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:20,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:23,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:23,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:23,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:23,446 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:23,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:25,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:26,450 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:28,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:28,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:28,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:29,860 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:29,861 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:29,861 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:29,866 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:30,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:30,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:32,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:34,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:34,898 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:34,898 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:34,903 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:35,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:35,904 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:36,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:38,315 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:53:39,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:39,046 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:39,047 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:39,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:39,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:40,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:41,052 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:43,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:43,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:43,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:44,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:44,396 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:44,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:44,402 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:44,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:46,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:46,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:51,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:55,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:55,254 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:55,254 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:55,259 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:55,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:57,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:58,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:58,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:58,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:58,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:00,283 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:00,284 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:00,284 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:00,290 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:00,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:02,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:02,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:07,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:07,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:07,353 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:07,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:07,359 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:07,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:08,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:54:10,496 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:13,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:13,139 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:13,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:13,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:13,145 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:13,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:13,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:13,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:13,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:14,501 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:17,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:17,677 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:17,678 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:17,683 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:18,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:18,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:20,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:22,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:22,323 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:22,323 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:22,328 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:22,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:24,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:24,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:27,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:27,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:27,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:27,701 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:28,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:28,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:28,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:28,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:29,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:30,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:33,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:33,249 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:33,249 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:33,254 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:33,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:35,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:36,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:38,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:54:40,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:40,672 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:40,673 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:40,674 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:40,678 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:41,526 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:42,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:43,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:43,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:43,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:45,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:46,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:46,357 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:46,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:46,363 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:46,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:48,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:50,848 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:50,849 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:50,849 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:50,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:50,855 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:51,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:52,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:54,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:54,781 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:54,781 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:54,786 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:55,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:56,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:56,787 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:58,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:58,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:58,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:59,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:59,457 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:59,457 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:59,462 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:59,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:02,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:02,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:05,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:05,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:05,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:05,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:05,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:06,554 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:08,027 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:08,321 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:55:09,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:09,135 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:09,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:09,142 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:09,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:10,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:13,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:13,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:13,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:13,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:14,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:14,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:14,436 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:14,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:14,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:16,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:18,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:21,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:21,450 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:21,451 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:21,456 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:21,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:24,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:24,572 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:26,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:26,975 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:26,975 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:26,980 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:27,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:28,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:28,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:28,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:28,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:30,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:31,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:31,228 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:31,228 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:31,233 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:31,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:32,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:36,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:37,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:37,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:37,212 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:37,217 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:37,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:38,330 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:55:38,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:41,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:41,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:41,580 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:41,580 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:41,587 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:41,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:43,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:43,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:43,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:44,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:46,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:46,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:46,690 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:46,690 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:46,695 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:47,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:48,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:51,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:51,254 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:51,254 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:51,259 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:51,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:52,259 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:52,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:55,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:55,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:55,743 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:55,749 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:56,603 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:57,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:58,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:58,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:58,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:58,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:00,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:00,433 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:00,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:00,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:00,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:02,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:03,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:06,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:06,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:06,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:06,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:06,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:08,325 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:56:08,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:09,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:10,939 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:10,940 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:10,940 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:10,945 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:11,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:12,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:13,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:13,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:13,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:14,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:14,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:14,893 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:14,893 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:14,898 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:15,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:16,628 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:19,934 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:21,091 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:21,092 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:21,092 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:21,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:21,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:22,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:25,098 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:27,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:27,793 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:27,794 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:27,799 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:28,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:28,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:28,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:28,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:30,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:30,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:34,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:34,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:34,779 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:34,784 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:35,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:35,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:36,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:38,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:56:41,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:43,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:43,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:43,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:43,744 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:43,745 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:43,745 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:43,751 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:44,652 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:46,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:46,752 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:47,825 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:47,825 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:47,826 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:47,831 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:48,655 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:50,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:51,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:52,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:52,568 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:52,568 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:52,573 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:52,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:54,663 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:56,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:56,418 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:56,418 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:56,424 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:56,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:57,424 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:58,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:58,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:58,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:58,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:01,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:01,780 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:01,781 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:01,786 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:02,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:02,787 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:04,674 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:06,220 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:06,220 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:06,221 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:06,226 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:06,677 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:08,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:08,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:57:08,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:10,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:10,726 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:10,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:10,732 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:11,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:12,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:13,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:13,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:13,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:13,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:15,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:15,128 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:15,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:15,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:15,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:16,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:19,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:19,732 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:19,733 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:19,733 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:19,740 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:20,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:22,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:24,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:25,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:25,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:25,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:25,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:25,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:28,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:28,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:28,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:28,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:30,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:31,419 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:31,420 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:31,420 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:31,425 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:31,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:34,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:35,462 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:35,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:35,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:35,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:35,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:36,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:38,332 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:57:38,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:40,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:40,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:40,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:40,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:40,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:41,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:42,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:43,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:43,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:43,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:45,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:45,050 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:45,050 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:45,055 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:45,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:46,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:47,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:50,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:50,294 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:50,294 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:50,299 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:50,730 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:52,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:52,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:54,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:54,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:54,787 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:54,792 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:55,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:56,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:57,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:58,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:58,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:58,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:59,803 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:59,804 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:59,804 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:59,809 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:00,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:02,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:02,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:05,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:05,089 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:05,089 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:05,094 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:05,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:06,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:08,095 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:08,337 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:58:10,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:10,835 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:10,836 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:10,841 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:11,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:12,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:13,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:13,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:13,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:13,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:18,304 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:18,304 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:18,305 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:18,310 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:18,760 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:19,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:20,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:23,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:23,301 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:23,302 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:23,307 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:23,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:24,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:25,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:28,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:28,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:28,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:30,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:30,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:30,403 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:30,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:30,408 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:30,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:32,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:35,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:36,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:36,347 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:36,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:36,353 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:36,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:38,337 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:58:38,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:41,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:43,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:43,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:43,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:44,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:44,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:44,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:44,181 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:44,787 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:46,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:47,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:52,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:52,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:52,453 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:52,453 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:52,460 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:52,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:54,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:57,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:57,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:57,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:57,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:57,318 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:57,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:58,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:58,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:58,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:58,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:02,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:03,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:03,541 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:03,541 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:03,546 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:03,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:04,808 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:07,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:08,339 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:59:10,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:10,495 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:10,496 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:10,500 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:10,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:12,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:13,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:13,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:13,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:13,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:15,799 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:15,800 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:15,800 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:15,806 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:15,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:18,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:18,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:21,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:21,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:21,247 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:21,251 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:21,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:22,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:24,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:25,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:25,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:25,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:25,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:25,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:26,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:28,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:28,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:28,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:29,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:30,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:30,250 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:30,250 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:30,255 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:30,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:32,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:34,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:34,614 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:34,615 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:34,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:34,620 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:34,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:36,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:38,341 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:59:38,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:38,985 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:38,986 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:38,991 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:39,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:39,991 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:40,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:43,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:43,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:43,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:43,486 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:43,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:43,488 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:43,493 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:43,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:44,850 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:45,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:48,848 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:48,849 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:48,849 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:48,855 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:48,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:50,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:50,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:54,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:54,890 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:54,891 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:54,896 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:55,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:55,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:56,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:58,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:58,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:58,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:59,446 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:59,447 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:59,447 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:59,452 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:59,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:00,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:01,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:04,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:04,568 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:04,568 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:04,573 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:04,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:06,574 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:06,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:08,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:00:10,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:10,990 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:10,991 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:10,995 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:11,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:11,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:12,884 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:13,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:13,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:13,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:16,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:16,115 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:16,115 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:16,120 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:16,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:17,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:18,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:21,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:21,610 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:21,611 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:21,616 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:21,892 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:22,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:22,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:26,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:26,994 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:26,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:27,000 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:27,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:28,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:28,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:28,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:28,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:28,905 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:31,927 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:31,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:31,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:31,934 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:32,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:33,935 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:34,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:36,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:36,433 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:36,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:36,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:36,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:38,346 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:00:38,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:39,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:43,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:43,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:43,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:44,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:45,078 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:45,079 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:45,079 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:45,084 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:45,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:46,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:50,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:50,880 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:50,881 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:50,881 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:50,887 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:50,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:52,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:55,291 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:55,292 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:55,293 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:55,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:55,298 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:55,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:56,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:58,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:58,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:58,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:00,591 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:00,592 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:00,592 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:00,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:00,598 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:00,936 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:02,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:05,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:05,446 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:05,446 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:05,451 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:05,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:06,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:06,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:08,348 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:01:12,273 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:12,274 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:12,275 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:12,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:12,280 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:12,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:13,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:13,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:13,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:14,953 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:16,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:16,535 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:16,536 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:16,540 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:16,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:17,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:18,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:20,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:20,457 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:20,457 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:20,462 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:20,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:22,961 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:23,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:28,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:28,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:28,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:28,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:28,994 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:28,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:28,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:28,999 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:29,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:30,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:34,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:34,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:34,351 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:34,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:34,358 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:34,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:36,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:38,353 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:01:39,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:40,879 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:40,880 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:40,881 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:40,886 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:40,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:42,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:43,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:43,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:43,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:44,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:45,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:45,338 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:45,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:45,343 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:45,985 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:46,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:50,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:50,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:50,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:50,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:50,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:50,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:52,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:55,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:57,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:57,403 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:57,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:57,408 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:57,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:58,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:58,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:58,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:58,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:00,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:03,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:03,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:03,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:03,100 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:04,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:05,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:06,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:08,352 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:02:08,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:08,389 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:08,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:08,394 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:09,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:11,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:11,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:13,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:13,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:13,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:16,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:16,840 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:16,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:16,841 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:16,846 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:17,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:19,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:21,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:21,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:21,312 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:21,317 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:22,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:22,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:23,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:25,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:25,411 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:25,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:25,416 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:26,029 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:27,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:27,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:28,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:28,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:28,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:31,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:31,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:31,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:31,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:32,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:33,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:33,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:35,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:35,577 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:35,577 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:35,582 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:36,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:37,047 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:38,354 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:02:38,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:40,907 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:40,908 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:40,908 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:40,913 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:41,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:43,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:43,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:43,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:43,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:44,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:49,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:49,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:49,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:49,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:49,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:50,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:51,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:53,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:53,833 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:53,833 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:53,838 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:54,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:54,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:57,077 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:58,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:58,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:58,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:58,476 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:58,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:58,494 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:58,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:59,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:00,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:01,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:05,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:08,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:08,184 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:08,184 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:08,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:08,357 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:03:09,092 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:11,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:11,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:13,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:13,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:13,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:13,430 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:13,430 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:13,435 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:14,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:15,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:16,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:17,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:17,923 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:17,923 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:17,928 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:18,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:21,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:21,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:23,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:23,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:23,703 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:23,708 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:24,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:25,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:27,708 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:28,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:28,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:28,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:29,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:29,488 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:29,488 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:29,493 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:30,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:31,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:33,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:36,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:36,404 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:36,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:36,409 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:37,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:38,359 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:03:39,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:39,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:41,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:41,472 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:41,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:41,477 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:42,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:43,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:43,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:43,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:43,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:44,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:47,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:47,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:47,246 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:47,251 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:48,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:49,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:50,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:52,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:52,731 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:52,731 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:52,736 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:53,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:55,167 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:55,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:58,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:58,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:58,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:01,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:01,270 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:01,270 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:01,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:01,275 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:02,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:03,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:06,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:06,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:06,030 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:06,035 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:06,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:07,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:08,365 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:04:09,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:12,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:12,958 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:12,959 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:12,959 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:12,964 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:13,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:13,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:13,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:13,195 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:15,202 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:17,454 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:18,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:18,488 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:18,488 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:18,493 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:19,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:21,213 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:22,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:23,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:23,822 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:23,822 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:23,827 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:24,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:25,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:27,827 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:28,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:28,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:28,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:33,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:33,156 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:33,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:33,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:33,162 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:33,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:34,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:38,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:38,363 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:04:40,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:40,815 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:40,815 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:40,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:41,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:42,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:43,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:43,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:43,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:43,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:46,108 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:46,109 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:46,109 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:46,114 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:46,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:48,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:49,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:52,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:52,001 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:52,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:52,006 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:52,264 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:54,269 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:55,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:57,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:57,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:57,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:57,216 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:57,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:58,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:58,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:58,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:58,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:00,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:03,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:03,086 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:03,086 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:03,091 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:03,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:04,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:06,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:07,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:07,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:07,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:07,333 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:08,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:08,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:08,372 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:05:11,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:11,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:11,974 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:11,974 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:11,979 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:12,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:13,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:13,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:13,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:14,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:16,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:16,927 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:16,928 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:16,928 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:16,933 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:17,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:18,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:21,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:21,123 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:21,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:21,128 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:21,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:22,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:22,316 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:26,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:26,216 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:26,216 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:26,221 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:26,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:27,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:28,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:28,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:28,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:28,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:32,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:35,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:35,331 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:35,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:35,336 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:35,336 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:36,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:38,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:38,368 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:05:40,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:40,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:40,644 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:40,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:41,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:42,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:43,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:43,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:43,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:43,419 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:47,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:47,479 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:47,479 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:47,484 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:48,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:48,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:48,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:52,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:52,830 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:52,830 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:52,835 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:53,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:53,836 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:54,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:58,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:58,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:58,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:58,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:58,485 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:58,485 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:58,490 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:59,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:59,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:00,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:02,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:02,911 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:02,911 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:02,916 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:03,376 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:04,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:04,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:07,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:07,173 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:07,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:07,178 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:07,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:08,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:06:08,386 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:10,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:13,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:13,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:13,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:13,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:13,413 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:13,413 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:13,418 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:14,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:14,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:15,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:19,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:19,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:19,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:19,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:19,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:20,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:21,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:24,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:24,582 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:24,582 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:24,587 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:25,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:26,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:26,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:28,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:28,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:28,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:31,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:31,018 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:31,018 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:31,023 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:31,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:32,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:32,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:36,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:36,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:36,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:36,177 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:36,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:37,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:38,376 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:06:38,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:40,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:40,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:40,643 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:40,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:41,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:42,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:42,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:43,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:43,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:43,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:44,815 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:44,815 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:44,816 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:44,821 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:45,438 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:46,442 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:47,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:48,725 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:48,726 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:48,726 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:48,731 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:49,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:50,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:53,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:53,429 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:53,430 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:53,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:53,434 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:53,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:54,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:58,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:58,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:58,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:59,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:59,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:59,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:59,873 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:59,878 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:00,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:00,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:04,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:04,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:04,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:04,946 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:04,950 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:05,467 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:06,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:08,375 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:07:09,488 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:09,489 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:09,489 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:09,494 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:10,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:10,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:10,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:13,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:13,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:13,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:14,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:14,331 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:14,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:14,336 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:14,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:16,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:16,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:19,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:19,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:19,659 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:19,664 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:20,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:20,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:21,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:24,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:24,107 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:24,108 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:24,112 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:24,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:26,501 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:27,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:28,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:28,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:28,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:28,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:28,556 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:28,556 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:28,561 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:29,503 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:30,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:32,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:32,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:32,887 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:32,887 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:32,892 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:33,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:34,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:37,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:38,380 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:07:38,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:38,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:38,474 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:38,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:38,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:40,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:43,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:43,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:43,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:43,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:43,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:43,423 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:43,423 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:43,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:43,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:44,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:48,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:51,508 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:51,509 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:51,509 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:51,515 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:51,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:52,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:53,515 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:55,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:55,791 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:55,791 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:55,796 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:56,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:56,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:58,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:58,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:58,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:59,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:00,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:00,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:00,281 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:00,286 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:00,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:02,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:05,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:05,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:05,549 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:05,549 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:05,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:05,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:06,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:08,379 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:08:10,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:10,336 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:10,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:10,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:10,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:10,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:12,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:13,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:13,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:13,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:15,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:15,152 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:15,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:15,157 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:15,572 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:16,193 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:16,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:21,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:21,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:21,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:21,465 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:21,470 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:21,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:22,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:26,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:26,001 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:26,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:26,006 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:26,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:26,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:27,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:28,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:28,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:28,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:30,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:30,625 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:30,626 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:30,630 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:31,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:32,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:32,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:35,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:35,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:35,702 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:35,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:36,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:36,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:37,708 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:38,381 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:08:39,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:39,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:39,970 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:39,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:40,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:40,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:42,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:43,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:43,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:43,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:45,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:45,319 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:45,320 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:45,324 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:45,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:46,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:48,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:50,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:50,329 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:50,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:50,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:50,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:52,630 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:54,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:57,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:57,374 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:57,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:57,379 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:57,633 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:58,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:58,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:58,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:58,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:59,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:01,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:01,606 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:01,606 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:01,611 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:01,640 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:02,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:04,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:08,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:08,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:08,023 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:08,028 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:08,383 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:09:08,652 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:08,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:10,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:12,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:12,556 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:12,557 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:12,561 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:12,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:13,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:13,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:13,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:14,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:15,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:16,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:16,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:16,778 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:16,783 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:17,663 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:18,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:20,822 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:21,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:21,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:21,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:21,519 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:21,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:22,674 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:26,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:27,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:27,763 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:27,763 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:27,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:28,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:28,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:28,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:28,682 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:28,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:31,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:31,989 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:31,990 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:31,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:31,995 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:32,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:32,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:37,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:38,385 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:09:38,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:38,464 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:38,464 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:38,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:38,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:40,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:42,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:42,760 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:42,761 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:42,762 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:42,767 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:43,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:43,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:43,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:43,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:44,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:48,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:48,406 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:48,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:48,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:48,412 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:48,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:50,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:53,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:53,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:53,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:53,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:53,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:53,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:54,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:57,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:57,714 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:57,715 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:57,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:57,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:58,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:58,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:58,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:58,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:58,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:02,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:02,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:02,482 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:02,487 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:02,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:03,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:04,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:06,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:06,962 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:06,962 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:06,967 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:07,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:08,388 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:10:08,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:09,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:13,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:13,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:13,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:13,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:13,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:13,459 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:13,464 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:13,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:14,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:14,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:19,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:20,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:20,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:20,436 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:20,441 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:20,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:22,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:24,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:25,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:25,468 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:25,469 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:25,476 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:25,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:26,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:28,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:28,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:28,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:30,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:30,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:30,479 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:30,479 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:30,484 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:30,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:32,776 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:35,296 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:35,297 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:35,297 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:35,302 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:35,778 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:36,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:36,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:38,390 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:10:39,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:39,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:39,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:39,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:39,784 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:40,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:41,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:43,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:43,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:43,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:45,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:45,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:45,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:45,014 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:45,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:46,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:47,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:50,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:50,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:50,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:50,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:51,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:52,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:52,820 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:56,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:56,683 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:56,684 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:56,689 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:56,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:58,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:58,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:58,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:58,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:58,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:02,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:02,224 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:02,225 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:02,229 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:02,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:02,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:04,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:06,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:06,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:06,664 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:06,669 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:06,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:08,395 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:11:08,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:09,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:11,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:11,160 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:11,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:11,165 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:11,831 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:12,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:13,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:13,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:13,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:15,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:16,979 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:16,980 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:16,980 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:16,985 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:17,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:18,844 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:21,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:21,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:21,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:21,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:21,283 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:21,847 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:22,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:25,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:25,382 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:25,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:25,388 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:25,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:26,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:26,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:28,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:28,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:28,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:29,894 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:29,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:29,895 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:29,900 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:30,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:30,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:31,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:34,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:34,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:34,013 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:34,019 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:34,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:34,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:37,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:38,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:11:42,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:42,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:42,791 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:42,792 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:42,799 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:42,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:43,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:43,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:43,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:44,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:47,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:48,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:48,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:48,872 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:48,877 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:48,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:50,892 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:52,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:53,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:53,364 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:53,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:53,370 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:53,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:54,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:58,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:58,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:58,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:58,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:58,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:58,628 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:58,629 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:58,634 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:58,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:00,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:03,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:03,145 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:03,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:03,151 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:03,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:04,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:04,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:08,399 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:12:09,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:09,324 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:09,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:09,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:09,330 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:09,917 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:10,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:13,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:13,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:13,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:14,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:14,194 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:14,195 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:14,199 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:14,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:14,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:15,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:18,891 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:18,892 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:18,892 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:18,897 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:18,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:20,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:20,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:25,486 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:25,486 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:25,487 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:25,491 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:25,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:26,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:26,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:28,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:28,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:28,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:32,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:32,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:32,196 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:32,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:32,202 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:32,954 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:32,954 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:37,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:37,129 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:37,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:37,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:37,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:38,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:38,399 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:12:38,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:41,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:41,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:41,437 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:41,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:41,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:42,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:43,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:43,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:43,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:45,935 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:45,936 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:45,936 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:45,941 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:45,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:46,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:48,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:50,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:50,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:50,066 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:50,071 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:50,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:50,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:54,071 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:55,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:55,584 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:55,585 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:55,589 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:55,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:56,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:58,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:58,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:58,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:59,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:59,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:59,409 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:59,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:59,414 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:59,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:00,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:04,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:05,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:05,822 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:05,822 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:05,828 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:06,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:07,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:08,401 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:13:10,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:10,107 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:10,107 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:10,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:10,112 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:11,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:11,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:13,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:13,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:13,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:14,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:14,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:14,767 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:14,772 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:15,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:15,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:17,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:19,193 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:19,194 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:19,194 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:19,199 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:20,022 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:21,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:21,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:23,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:23,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:23,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:23,892 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:24,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:25,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:26,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:28,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:28,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:28,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:28,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:28,961 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:28,961 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:28,966 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:29,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:31,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:32,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:33,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:33,899 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:33,900 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:33,905 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:34,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:35,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:37,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:38,403 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:13:38,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:38,438 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:38,439 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:38,444 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:39,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:39,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:43,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:43,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:43,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:43,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:44,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:44,140 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:44,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:44,146 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:45,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:45,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:49,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:50,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:50,002 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:50,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:50,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:50,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:51,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:54,076 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:54,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:54,077 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:54,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:55,077 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:55,077 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:55,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:58,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:58,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:58,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:59,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:59,958 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:59,958 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:59,963 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:00,081 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:01,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:01,086 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:06,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:08,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:08,329 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:08,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:08,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:08,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:14:09,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:09,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:11,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:13,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:13,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:13,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:14,499 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:14,500 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:14,500 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:14,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:15,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:15,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:16,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:19,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:19,345 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:19,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:19,350 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:20,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:21,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:22,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:24,031 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:24,032 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:24,032 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:24,037 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:24,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:25,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:28,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:28,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:28,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:28,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:29,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:29,294 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:29,294 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:29,299 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:30,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:31,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:33,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:34,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:34,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:34,821 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:34,826 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:35,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:37,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:38,420 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:14:38,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:43,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:43,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:43,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:43,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:43,463 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:43,464 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:43,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:43,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:44,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:45,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:48,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:49,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:49,219 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:49,219 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:49,224 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:50,155 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:51,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:53,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:53,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:53,775 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:53,775 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:53,780 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:54,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:55,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:58,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:58,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:58,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:59,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:59,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:59,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:59,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:59,339 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:00,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:01,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:04,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:06,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:06,552 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:06,552 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:06,557 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:07,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:07,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:08,410 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:15:09,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:13,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:13,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:13,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:13,444 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:13,445 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:13,449 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:14,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:14,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:15,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:19,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:21,826 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:21,827 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:21,827 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:21,832 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:22,202 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:23,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:24,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:28,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:28,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:28,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:30,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:30,140 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:30,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:30,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:30,145 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:30,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:31,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:34,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:34,443 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:34,443 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:34,448 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:35,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:35,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:35,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:38,412 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:15:39,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:39,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:39,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:39,027 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:39,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:41,027 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:41,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:43,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:43,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:43,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:44,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:44,073 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:44,074 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:44,079 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:44,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:45,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:46,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:48,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:48,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:48,640 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:48,645 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:49,245 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:49,246 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:51,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:56,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:57,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:57,237 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:57,237 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:57,242 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:57,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:58,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:58,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:58,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:59,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:02,415 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:03,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:03,955 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:03,955 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:03,960 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:04,264 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:05,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:07,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:08,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:16:09,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:09,610 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:09,610 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:09,615 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:10,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:11,276 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:13,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:13,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:13,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:13,480 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:16,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:16,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:16,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:16,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:17,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:17,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:18,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:21,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:21,571 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:21,571 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:21,576 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:22,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:23,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:24,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:28,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:28,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:28,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:29,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:29,857 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:29,858 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:29,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:29,863 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:30,303 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:31,307 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:34,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:35,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:35,171 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:35,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:35,177 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:35,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:37,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:38,416 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:16:40,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:41,736 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:41,737 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:41,737 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:41,742 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:42,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:43,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:43,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:43,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:43,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:45,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:46,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:46,556 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:46,557 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:46,563 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:47,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:47,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:50,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:50,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:50,990 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:50,990 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:50,995 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:51,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:53,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:55,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:58,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:58,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:58,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:59,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:59,417 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:59,417 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:59,422 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:00,346 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:01,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:01,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:03,482 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:03,483 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:03,483 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:03,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:04,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:05,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:06,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:08,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:08,135 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:08,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:08,141 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:08,359 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:08,422 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:17:09,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:12,424 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:12,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:12,873 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:12,873 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:12,878 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:13,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:13,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:13,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:13,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:15,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:18,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:21,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:21,050 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:21,050 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:21,055 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:21,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:23,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:24,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:27,523 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:27,524 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:27,524 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:27,529 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:28,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:28,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:28,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:28,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:29,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:29,424 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:32,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:32,239 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:32,240 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:32,244 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:32,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:33,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:35,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:38,421 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:17:40,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:40,748 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:40,748 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:40,749 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:40,754 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:41,409 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:41,409 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:43,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:43,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:43,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:45,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:49,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:49,051 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:49,052 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:49,057 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:49,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:51,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:51,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:54,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:54,114 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:54,114 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:54,120 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:54,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:55,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:56,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:58,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:58,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:58,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:01,471 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:01,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:01,476 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:01,476 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:01,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:02,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:03,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:05,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:05,713 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:05,714 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:05,718 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:06,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:06,719 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:07,449 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:08,426 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:18:11,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:11,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:11,020 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:11,025 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:11,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:12,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:13,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:13,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:13,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:13,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:15,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:15,800 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:15,801 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:15,806 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:16,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:17,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:17,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:22,842 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:23,718 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:23,718 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:23,719 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:23,724 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:24,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:25,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:28,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:28,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:28,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:28,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:28,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:28,686 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:28,686 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:28,691 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:29,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:29,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:33,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:33,430 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:33,431 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:33,431 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:33,436 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:33,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:35,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:38,425 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:18:39,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:42,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:42,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:42,095 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:42,099 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:42,496 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:43,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:43,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:43,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:43,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:44,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:47,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:47,074 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:47,074 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:47,079 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:47,505 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:49,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:50,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:52,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:52,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:52,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:52,348 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:52,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:53,516 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:55,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:56,866 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:56,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:56,867 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:56,871 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:57,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:57,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:58,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:58,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:58,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:00,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:01,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:01,390 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:01,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:01,395 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:01,526 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:03,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:05,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:06,803 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:06,804 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:06,804 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:06,809 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:07,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:07,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:08,427 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:19:11,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:11,134 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:11,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:11,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:11,139 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:11,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:13,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:13,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:13,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:13,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:16,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:17,111 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:17,112 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:17,112 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:17,117 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:17,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:19,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:21,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:21,313 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:21,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:21,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:21,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:22,319 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:23,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:25,971 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:25,971 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:25,972 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:25,977 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:26,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:27,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:27,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:28,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:28,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:28,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:29,830 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:29,831 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:29,831 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:29,836 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:30,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:31,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:33,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:34,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:34,834 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:34,835 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:34,845 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:35,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:35,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:38,430 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:19:38,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:38,785 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:38,786 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:38,791 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:39,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:39,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:39,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:43,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:43,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:43,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:43,672 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:43,673 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:43,673 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:43,678 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:44,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:45,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:45,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:49,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:49,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:49,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:49,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:49,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:51,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:51,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:54,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:54,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:54,790 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:54,796 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:55,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:55,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:56,796 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:58,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:58,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:58,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:00,496 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:00,497 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:00,497 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:00,502 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:00,589 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:01,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:02,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:06,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:06,051 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:06,051 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:06,056 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:06,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:07,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:08,056 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:08,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:20:12,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:12,882 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:12,882 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:12,887 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:13,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:13,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:13,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:13,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:13,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:13,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:16,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:16,937 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:16,937 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:16,942 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:17,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:17,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:18,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:22,231 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:22,231 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:22,232 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:22,236 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:22,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:23,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:24,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:27,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:27,627 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:27,628 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:27,632 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:28,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:28,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:28,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:28,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:29,480 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:29,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:32,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:32,259 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:32,259 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:32,265 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:32,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:33,624 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:35,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:36,535 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:36,536 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:36,537 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:36,541 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:36,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:37,628 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:38,437 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:20:40,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:41,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:41,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:41,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:41,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:41,630 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:43,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:43,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:43,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:43,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:46,027 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:46,028 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:46,028 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:46,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:46,033 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:46,636 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:47,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:50,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:50,459 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:50,460 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:50,465 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:50,640 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:51,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:51,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:56,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:56,445 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:56,446 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:56,452 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:56,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:57,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:57,648 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:58,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:58,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:58,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:01,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:01,182 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:01,182 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:01,187 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:01,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:03,225 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:03,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:08,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:08,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:21:09,524 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:09,525 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:09,525 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:09,530 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:09,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:11,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:13,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:13,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:13,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:13,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:14,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:14,064 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:14,064 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:14,069 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:14,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:15,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:18,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:18,678 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:18,678 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:18,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:18,684 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:19,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:19,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:23,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:27,822 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:27,823 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:27,823 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:27,828 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:28,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:28,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:28,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:28,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:29,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:29,680 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:31,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:31,885 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:31,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:31,890 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:32,682 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:33,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:34,891 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:38,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:38,100 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:38,101 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:38,105 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:38,438 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:21:38,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:39,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:40,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:43,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:43,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:43,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:44,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:44,396 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:44,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:44,402 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:44,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:45,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:46,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:48,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:48,656 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:48,656 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:48,661 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:48,699 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:49,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:51,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:53,931 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:53,932 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:53,932 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:53,937 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:54,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:55,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:56,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:58,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:58,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:58,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:58,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:58,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:58,502 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:58,506 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:58,707 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:59,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:02,507 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:04,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:04,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:04,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:04,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:04,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:05,715 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:08,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:08,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:22:09,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:09,635 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:09,635 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:09,640 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:09,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:11,722 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:13,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:13,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:13,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:15,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:15,998 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:15,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:16,003 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:16,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:17,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:19,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:21,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:21,019 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:21,019 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:21,024 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:21,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:21,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:25,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:28,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:28,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:28,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:30,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:30,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:30,312 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:30,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:30,317 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:30,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:31,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:34,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:34,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:34,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:34,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:35,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:35,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:35,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:38,442 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:22:39,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:39,066 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:39,066 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:39,071 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:39,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:39,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:41,072 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:43,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:43,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:43,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:43,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:43,407 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:43,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:43,413 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:43,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:45,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:46,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:49,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:49,857 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:49,857 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:49,862 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:50,760 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:51,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:51,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:54,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:54,129 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:54,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:54,137 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:54,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:55,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:57,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:58,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:58,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:58,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:58,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:58,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:58,162 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:58,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:58,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:59,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:02,468 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:04,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:04,277 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:04,277 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:04,282 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:04,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:05,775 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:08,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:08,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:23:08,991 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:08,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:08,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:08,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:09,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:09,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:13,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:13,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:13,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:14,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:14,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:14,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:14,823 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:15,787 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:15,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:18,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:19,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:19,095 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:19,096 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:19,101 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:19,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:19,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:23,767 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:23,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:23,768 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:23,773 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:23,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:24,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:25,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:28,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:28,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:28,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:28,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:28,502 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:28,502 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:28,507 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:28,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:29,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:30,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:33,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:33,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:33,778 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:33,783 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:33,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:35,783 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:35,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:38,447 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:23:38,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:38,625 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:38,625 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:38,630 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:38,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:39,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:41,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:42,696 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:42,696 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:42,697 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:42,702 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:42,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:43,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:43,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:43,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:43,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:47,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:49,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:49,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:49,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:49,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:49,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:51,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:53,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:53,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:53,803 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:53,803 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:53,808 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:53,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:55,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:58,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:58,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:58,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:58,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:59,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:59,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:59,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:59,333 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:59,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:01,832 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:04,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:04,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:04,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:04,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:04,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:04,834 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:05,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:08,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:08,082 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:08,083 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:08,088 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:08,455 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:24:08,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:09,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:09,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:13,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:13,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:13,024 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:13,030 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:13,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:13,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:13,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:13,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:13,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:14,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:17,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:17,318 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:17,318 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:17,324 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:17,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:19,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:20,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:24,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:24,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:24,047 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:24,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:24,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:25,860 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:26,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:28,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:28,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:28,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:29,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:29,786 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:29,787 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:29,791 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:29,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:31,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:31,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:34,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:34,201 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:34,201 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:34,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:34,867 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:35,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:37,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:38,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:24:39,259 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:39,260 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:39,261 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:39,266 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:39,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:39,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:42,266 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:43,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:43,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:43,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:46,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:46,428 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:46,428 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:46,432 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:46,879 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:47,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:47,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:51,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:51,377 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:51,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:51,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:51,884 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:53,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:53,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:57,096 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:57,097 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:57,097 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:57,102 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:57,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:57,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:58,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:58,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:58,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:58,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:02,096 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:02,097 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:02,097 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:02,102 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:02,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:03,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:04,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:06,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:06,386 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:06,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:06,391 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:06,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:07,905 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:08,457 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:25:09,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:10,538 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:10,539 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:10,539 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:10,544 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:10,908 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:11,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:13,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:13,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:13,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:15,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:15,184 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:15,184 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:15,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:15,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:15,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:15,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:19,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:19,906 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:19,906 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:19,911 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:19,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:20,911 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:21,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:25,912 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:28,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:28,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:28,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:28,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:28,981 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:28,982 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:28,986 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:29,928 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:29,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:31,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:35,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:35,344 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:35,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:35,350 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:35,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:35,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:36,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:38,456 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:25:39,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:39,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:39,322 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:39,327 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:39,938 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:39,938 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:42,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:43,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:43,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:43,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:43,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:43,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:43,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:43,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:44,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:45,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:48,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:48,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:48,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:48,058 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:48,063 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:48,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:49,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:53,064 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:53,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:53,676 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:53,676 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:53,681 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:53,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:55,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:57,991 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:57,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:57,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:57,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:58,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:58,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:58,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:58,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:58,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:59,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:03,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:03,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:03,946 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:03,946 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:03,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:03,961 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:05,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:08,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:26:08,460 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:09,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:09,858 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:09,859 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:09,864 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:09,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:11,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:13,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:13,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:13,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:14,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:14,046 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:14,047 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:14,047 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:14,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:14,974 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:15,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:18,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:18,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:18,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:18,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:18,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:19,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:19,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:23,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:23,306 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:23,306 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:23,312 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:23,984 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:23,984 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:25,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:28,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:28,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:28,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:29,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:29,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:29,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:29,519 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:29,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:30,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:31,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:34,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:34,893 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:34,893 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:34,898 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:34,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:35,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:35,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:38,460 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:26:40,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:40,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:40,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:40,016 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:40,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:41,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:42,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:43,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:43,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:43,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:44,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:44,615 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:44,615 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:44,621 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:45,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:46,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:46,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:50,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:50,137 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:50,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:50,142 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:51,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:52,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:52,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:54,803 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:54,804 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:54,804 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:54,809 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:55,019 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:56,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:57,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:58,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:58,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:58,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:00,708 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:00,708 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:00,709 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:00,714 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:01,024 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:02,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:03,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:05,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:05,426 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:05,426 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:05,431 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:06,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:06,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:08,462 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:27:09,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:10,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:10,005 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:10,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:10,011 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:10,035 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:12,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:13,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:13,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:13,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:15,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:16,972 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:16,973 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:16,973 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:16,978 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:17,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:18,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:21,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:21,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:21,454 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:21,454 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:21,459 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:22,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:22,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:26,460 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:27,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:27,913 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:27,913 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:27,918 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:28,053 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:28,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:28,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:28,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:30,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:32,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:32,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:32,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:32,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:32,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:33,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:34,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:36,788 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:36,788 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:36,789 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:36,794 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:37,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:37,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:38,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:38,468 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:27:43,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:43,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:43,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:43,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:43,422 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:43,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:43,422 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:43,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:44,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:44,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:48,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:48,358 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:48,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:48,363 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:49,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:49,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:50,077 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:52,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:52,810 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:52,810 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:52,816 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:53,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:54,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:54,816 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:58,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:58,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:58,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:58,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:58,541 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:58,542 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:58,547 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:59,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:00,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:00,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:04,852 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:04,853 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:04,853 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:04,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:05,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:05,859 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:06,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:08,467 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:28:10,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:10,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:10,288 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:10,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:11,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:11,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:12,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:13,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:13,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:13,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:16,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:16,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:16,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:16,515 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:16,520 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:17,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:18,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:20,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:20,546 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:20,546 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:20,551 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:21,109 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:21,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:22,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:25,219 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:25,220 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:25,220 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:25,225 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:26,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:26,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:27,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:28,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:28,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:28,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:29,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:29,542 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:29,542 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:29,548 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:30,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:32,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:32,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:36,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:36,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:36,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:36,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:37,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:38,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:38,472 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:28:38,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:43,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:43,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:43,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:43,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:43,555 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:43,556 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:43,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:43,561 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:44,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:46,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:47,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:47,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:47,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:47,919 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:48,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:48,919 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:50,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:52,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:52,644 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:52,644 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:52,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:53,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:54,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:54,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:57,752 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:57,753 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:57,753 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:57,758 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:58,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:58,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:58,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:58,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:00,147 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:00,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:04,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:04,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:04,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:04,845 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:05,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:05,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:06,155 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:08,471 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:29:09,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:09,100 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:09,101 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:09,106 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:09,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:10,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:11,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:13,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:13,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:13,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:14,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:14,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:14,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:14,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:15,165 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:16,165 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:16,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:20,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:20,857 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:20,857 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:20,862 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:21,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:21,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:22,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:25,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:25,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:25,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:25,934 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:26,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:26,935 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:28,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:28,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:28,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:28,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:31,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:31,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:31,030 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:31,036 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:31,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:32,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:32,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:37,074 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:38,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:38,201 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:38,201 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:38,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:38,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:29:39,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:40,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:42,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:43,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:43,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:43,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:43,676 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:43,676 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:43,681 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:44,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:46,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:47,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:47,772 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:47,773 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:47,773 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:47,779 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:48,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:50,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:52,656 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:52,657 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:52,657 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:52,662 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:53,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:53,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:54,202 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:58,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:58,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:58,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:58,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:58,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:58,466 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:58,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:59,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:59,471 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:00,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:02,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:02,467 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:02,467 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:02,472 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:03,209 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:04,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:04,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:07,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:07,119 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:07,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:07,125 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:07,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:08,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:08,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:30:09,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:11,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:11,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:11,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:11,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:12,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:12,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:13,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:13,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:13,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:15,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:16,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:16,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:16,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:16,891 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:17,226 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:18,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:20,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:22,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:22,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:22,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:22,322 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:23,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:24,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:26,323 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:27,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:27,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:27,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:27,012 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:27,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:28,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:28,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:28,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:28,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:31,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:31,303 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:31,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:31,307 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:31,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:32,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:32,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:36,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:36,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:36,243 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:36,247 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:36,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:37,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:38,250 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:38,478 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:30:42,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:42,338 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:42,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:42,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:42,343 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:43,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:43,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:43,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:43,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:44,256 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:46,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:46,454 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:46,454 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:46,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:47,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:47,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:48,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:52,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:55,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:55,207 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:55,207 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:55,212 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:55,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:56,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:58,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:58,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:58,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:58,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:59,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:59,457 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:59,457 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:59,462 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:00,272 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:00,272 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:03,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:05,011 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:05,012 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:05,012 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:05,017 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:05,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:06,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:08,483 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:31:09,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:09,382 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:09,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:09,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:09,386 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:10,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:10,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:13,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:13,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:13,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:13,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:13,652 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:13,652 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:13,657 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:14,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:14,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:14,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:19,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:19,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:19,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:19,642 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:20,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:20,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:20,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:24,760 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:24,761 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:24,761 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:24,766 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:25,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:25,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:26,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:28,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:28,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:28,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:29,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:29,303 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:29,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:29,308 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:30,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:30,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:31,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:33,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:33,554 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:33,554 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:33,559 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:34,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:34,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:36,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:38,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:38,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:38,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:38,439 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:38,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:31:39,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:40,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:42,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:43,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:43,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:43,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:43,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:43,463 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:43,463 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:43,468 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:44,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:44,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:48,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:51,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:51,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:51,870 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:51,875 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:52,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:53,876 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:54,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:57,204 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:57,205 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:57,205 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:57,210 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:57,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:58,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:58,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:58,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:58,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:59,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:02,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:02,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:02,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:02,918 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:03,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:04,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:04,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:07,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:07,530 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:07,530 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:07,535 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:08,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:08,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:08,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:32:10,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:13,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:13,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:13,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:13,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:13,772 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:13,773 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:13,778 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:14,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:14,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:15,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:20,779 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:20,825 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:20,826 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:20,826 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:20,831 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:21,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:22,357 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:25,831 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:25,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:25,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:25,976 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:25,981 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:26,359 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:28,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:28,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:28,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:28,362 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:30,528 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:30,529 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:30,529 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:30,533 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:31,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:31,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:32,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:35,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:35,391 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:35,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:35,396 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:36,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:36,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:37,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:38,486 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:32:42,112 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:42,113 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:42,113 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:42,118 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:42,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:43,119 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:43,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:43,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:43,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:44,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:48,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:50,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:50,478 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:50,478 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:50,483 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:51,385 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:52,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:53,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:54,259 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:54,260 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:54,260 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:54,265 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:54,388 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:56,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:58,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:58,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:58,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:59,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:59,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:59,691 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:59,691 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:59,696 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:00,396 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:00,396 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:04,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:04,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:04,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:04,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:04,437 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:05,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:06,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:08,489 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:33:09,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:13,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:13,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:13,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:13,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:13,683 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:13,683 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:13,688 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:14,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:14,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:14,725 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:18,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:18,295 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:18,295 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:18,300 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:18,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:20,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:20,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:24,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:24,979 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:24,979 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:24,984 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:25,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:25,984 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:26,422 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:28,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:28,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:28,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:30,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:30,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:30,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:30,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:30,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:31,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:32,427 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:35,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:35,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:35,212 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:35,217 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:35,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:36,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:37,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:38,490 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:33:41,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:41,140 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:41,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:41,146 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:41,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:42,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:43,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:43,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:43,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:45,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:45,626 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:45,626 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:45,631 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:46,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:46,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:48,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:51,840 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:51,841 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:51,841 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:51,846 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:52,445 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:52,445 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:53,847 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:57,231 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:57,232 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:57,232 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:57,237 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:57,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:58,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:58,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:58,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:58,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:59,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:01,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:01,427 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:01,427 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:01,432 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:01,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:02,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:04,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:05,918 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:05,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:05,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:05,924 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:06,458 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:06,458 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:08,493 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:34:09,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:10,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:10,283 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:10,283 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:10,288 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:10,462 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:12,465 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:13,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:13,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:15,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:17,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:17,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:17,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:17,949 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:18,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:18,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:20,950 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:22,932 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:22,933 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:22,933 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:22,938 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:23,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:24,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:26,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:26,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:26,878 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:26,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:26,883 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:27,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:28,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:28,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:28,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:28,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:32,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:34,486 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:34,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:34,487 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:34,496 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:35,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:36,488 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:37,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:38,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:34:38,783 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:38,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:38,784 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:38,789 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:39,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:40,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:42,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:43,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:43,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:43,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:45,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:45,404 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:45,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:45,409 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:45,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:46,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:48,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:50,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:50,329 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:50,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:50,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:50,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:52,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:54,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:55,696 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:55,697 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:55,697 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:55,702 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:56,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:56,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:58,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:58,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:58,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:59,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:04,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:04,125 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:04,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:04,130 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:04,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:05,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:06,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:08,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:08,192 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:08,193 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:08,199 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:08,497 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:35:08,526 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:10,499 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:10,530 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:13,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:13,738 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:13,739 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:13,739 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:13,744 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:14,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:14,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:15,744 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:17,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:17,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:17,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:17,819 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:18,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:18,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:20,820 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:25,820 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:26,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:26,167 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:26,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:26,172 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:26,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:28,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:28,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:28,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:28,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:31,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:32,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:32,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:32,024 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:32,029 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:32,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:32,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:37,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:38,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:38,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:35:38,502 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:38,502 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:38,506 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:38,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:40,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:42,507 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:43,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:43,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:43,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:45,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:45,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:45,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:45,478 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:45,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:46,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:48,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:49,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:49,768 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:49,768 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:49,773 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:50,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:50,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:53,774 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:54,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:54,671 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:54,671 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:54,676 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:55,572 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:56,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:58,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:58,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:58,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:59,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:59,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:59,581 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:59,581 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:59,586 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:00,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:00,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:04,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:04,587 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:04,587 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:04,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:04,592 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:05,581 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:06,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:08,502 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:36:09,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:09,845 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:09,846 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:09,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:09,850 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:10,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:10,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:13,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:14,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:14,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:14,058 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:14,063 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:14,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:14,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:15,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:19,782 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:19,782 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:19,782 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:19,787 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:20,600 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:20,600 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:20,788 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:24,830 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:24,831 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:24,831 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:24,836 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:25,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:25,836 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:26,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:28,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:28,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:28,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:29,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:29,106 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:29,106 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:29,111 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:29,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:30,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:31,111 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:33,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:33,573 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:33,573 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:33,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:33,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:34,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:36,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:38,504 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:36:38,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:38,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:38,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:38,869 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:39,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:40,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:41,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:43,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:43,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:43,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:45,528 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:45,528 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:45,529 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:45,534 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:45,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:46,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:47,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:52,535 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:54,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:54,365 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:54,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:54,371 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:54,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:56,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:58,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:58,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:58,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:58,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:03,419 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:03,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:03,696 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:03,696 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:03,701 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:04,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:06,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:08,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:08,354 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:08,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:08,359 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:08,510 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:37:08,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:08,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:10,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:12,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:12,651 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:12,651 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:12,656 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:12,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:13,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:14,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:14,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:18,915 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:18,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:18,916 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:18,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:19,663 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:19,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:20,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:24,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:26,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:26,042 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:26,042 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:26,048 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:26,669 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:28,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:28,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:28,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:28,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:30,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:30,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:30,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:30,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:30,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:30,674 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:32,677 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:35,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:35,504 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:35,504 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:35,504 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:35,509 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:35,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:36,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:38,509 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:37:40,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:40,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:40,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:40,898 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:40,902 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:41,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:42,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:43,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:43,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:43,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:46,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:46,508 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:46,509 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:46,509 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:46,514 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:46,690 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:48,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:51,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:54,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:54,873 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:54,873 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:54,878 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:55,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:56,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:56,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:58,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:58,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:58,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:59,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:59,396 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:59,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:59,401 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:59,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:00,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:02,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:03,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:03,699 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:03,699 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:03,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:03,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:04,707 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:07,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:08,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:38:09,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:09,730 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:09,730 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:09,735 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:10,715 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:10,715 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:12,735 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:13,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:13,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:13,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:14,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:14,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:14,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:14,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:14,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:16,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:18,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:22,654 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:22,655 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:22,655 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:22,660 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:22,724 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:23,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:24,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:27,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:27,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:27,554 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:27,559 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:27,730 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:28,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:28,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:28,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:28,733 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:29,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:32,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:32,088 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:32,089 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:32,093 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:32,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:34,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:35,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:38,513 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:38:40,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:40,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:40,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:40,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:40,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:40,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:42,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:43,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:43,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:43,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:44,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:44,846 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:44,846 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:44,851 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:45,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:45,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:46,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:49,515 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:49,516 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:49,516 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:49,521 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:49,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:50,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:51,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:54,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:54,585 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:54,586 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:54,590 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:54,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:56,591 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:56,759 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:58,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:58,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:58,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:00,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:00,311 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:00,311 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:00,316 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:00,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:02,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:02,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:05,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:05,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:05,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:05,322 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:05,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:06,769 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:07,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:08,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:39:10,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:10,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:10,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:10,177 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:10,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:12,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:13,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:13,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:13,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:13,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:14,738 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:14,739 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:14,739 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:14,745 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:14,778 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:16,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:18,746 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:20,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:20,507 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:20,507 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:20,511 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:20,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:22,787 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:24,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:25,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:25,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:25,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:25,715 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:25,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:26,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:28,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:28,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:28,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:29,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:29,965 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:29,966 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:29,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:29,971 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:30,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:32,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:34,971 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:35,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:35,131 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:35,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:35,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:35,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:36,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:38,517 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:39:40,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:40,360 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:40,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:40,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:40,365 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:40,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:42,808 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:43,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:43,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:43,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:45,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:45,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:45,470 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:45,470 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:45,475 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:45,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:46,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:50,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:50,812 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:50,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:50,813 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:50,818 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:50,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:52,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:55,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:56,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:56,323 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:56,323 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:56,329 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:56,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:58,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:58,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:58,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:58,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:01,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:02,027 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:02,028 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:02,028 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:02,033 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:02,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:04,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:06,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:06,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:06,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:06,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:06,573 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:06,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:08,519 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:40:08,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:11,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:11,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:11,790 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:11,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:11,796 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:11,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:12,844 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:13,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:13,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:13,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:17,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:17,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:17,559 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:17,559 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:17,564 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:17,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:18,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:22,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:23,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:23,430 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:23,431 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:23,435 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:23,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:24,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:28,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:28,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:28,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:28,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:29,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:29,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:29,186 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:29,191 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:29,860 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:30,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:34,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:34,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:34,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:34,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:34,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:34,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:36,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:38,525 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:40:39,499 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:39,500 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:39,500 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:39,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:39,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:39,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:40,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:43,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:43,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:43,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:45,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:47,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:47,846 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:47,847 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:47,851 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:47,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:48,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:50,852 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:53,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:53,091 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:53,091 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:53,096 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:53,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:54,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:56,096 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:58,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:58,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:58,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:00,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:00,272 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:00,272 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:00,277 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:00,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:01,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:02,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:04,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:04,723 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:04,723 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:04,728 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:04,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:06,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:06,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:08,524 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:41:09,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:09,158 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:09,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:09,163 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:09,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:10,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:12,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:13,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:13,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:13,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:13,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:13,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:13,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:13,823 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:13,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:14,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:17,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:21,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:21,680 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:21,680 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:21,685 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:21,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:22,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:23,686 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:26,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:26,404 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:26,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:26,409 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:26,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:28,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:28,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:28,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:28,919 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:29,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:31,483 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:31,483 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:31,483 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:31,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:31,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:32,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:34,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:36,772 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:36,773 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:36,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:36,778 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:36,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:38,529 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:41:38,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:39,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:42,060 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:42,060 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:42,060 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:42,065 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:42,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:43,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:43,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:43,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:44,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:45,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:46,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:46,621 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:46,621 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:46,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:46,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:48,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:50,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:53,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:53,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:53,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:53,177 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:53,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:54,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:56,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:58,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:58,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:58,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:58,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:58,717 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:58,717 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:58,722 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:58,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:00,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:01,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:03,048 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:03,049 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:03,050 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:03,055 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:03,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:04,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:07,055 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:08,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:08,181 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:08,181 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:08,186 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:08,528 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:42:08,961 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:10,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:12,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:13,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:13,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:13,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:17,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:17,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:17,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:17,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:17,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:18,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:18,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:22,087 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:22,088 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:22,089 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:22,094 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:22,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:24,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:24,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:25,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:25,954 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:25,955 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:25,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:25,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:26,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:28,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:28,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:28,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:29,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:31,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:31,418 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:31,418 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:31,423 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:31,985 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:32,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:34,424 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:35,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:35,911 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:35,911 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:35,917 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:35,992 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:36,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:38,530 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:42:39,532 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:41,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:41,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:41,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:41,159 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:41,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:43,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:43,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:43,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:43,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:45,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:45,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:45,631 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:45,631 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:45,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:46,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:47,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:50,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:54,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:54,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:54,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:54,173 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:55,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:56,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:57,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:58,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:58,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:58,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:58,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:58,387 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:58,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:58,392 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:59,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:01,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:01,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:03,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:03,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:03,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:03,553 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:04,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:05,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:06,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:08,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:08,208 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:08,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:08,214 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:08,533 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:43:09,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:11,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:12,460 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:12,461 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:12,461 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:12,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:12,466 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:13,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:13,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:13,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:13,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:15,035 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:17,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:17,316 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:17,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:17,321 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:18,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:18,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:19,040 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:21,768 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:21,769 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:21,769 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:21,774 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:22,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:23,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:23,775 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:27,176 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:27,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:27,177 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:27,181 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:28,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:28,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:28,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:28,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:29,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:29,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:31,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:31,275 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:31,276 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:31,281 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:32,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:33,053 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:35,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:36,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:36,454 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:36,454 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:36,459 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:37,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:38,535 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:43:39,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:40,536 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:43,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:43,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:43,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:44,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:44,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:44,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:44,081 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:45,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:45,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:46,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:48,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:48,835 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:48,835 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:48,840 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:49,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:51,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:51,840 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:56,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:58,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:58,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:58,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:58,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:58,431 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:58,431 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:58,436 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:59,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:01,081 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:02,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:02,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:02,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:02,459 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:02,464 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:03,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:05,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:06,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:06,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:06,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:06,732 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:07,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:07,732 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:08,541 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:44:09,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:10,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:10,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:10,979 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:10,984 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:11,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:12,984 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:13,096 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:13,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:13,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:13,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:16,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:16,336 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:16,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:16,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:17,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:18,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:19,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:20,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:20,493 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:20,493 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:20,498 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:21,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:23,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:23,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:26,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:26,256 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:26,256 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:26,262 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:27,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:28,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:28,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:28,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:29,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:29,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:31,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:31,064 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:31,064 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:31,069 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:31,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:33,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:35,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:38,539 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:44:39,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:39,374 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:39,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:39,379 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:40,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:40,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:41,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:43,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:43,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:43,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:43,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:43,874 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:43,875 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:43,880 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:44,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:45,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:45,880 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:50,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:50,377 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:50,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:50,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:51,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:51,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:53,137 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:54,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:54,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:54,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:54,889 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:55,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:56,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:57,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:58,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:58,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:58,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:00,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:00,225 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:00,225 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:00,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:01,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:01,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:02,231 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:05,232 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:05,233 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:05,234 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:05,238 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:06,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:07,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:07,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:08,545 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:45:12,546 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:13,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:13,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:13,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:13,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:13,637 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:13,637 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:13,642 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:14,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:15,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:17,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:18,111 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:18,111 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:18,112 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:18,117 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:18,162 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:19,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:22,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:22,565 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:22,565 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:22,570 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:23,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:23,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:25,169 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:28,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:28,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:28,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:29,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:31,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:31,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:31,033 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:31,039 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:31,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:33,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:35,040 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:37,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:37,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:37,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:37,216 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:38,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:38,544 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:45:39,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:40,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:41,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:41,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:41,703 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:41,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:42,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:43,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:43,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:43,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:43,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:46,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:47,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:47,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:47,650 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:47,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:48,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:49,192 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:51,656 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:55,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:55,462 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:55,462 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:55,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:56,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:57,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:57,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:58,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:58,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:58,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:02,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:02,228 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:02,228 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:02,234 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:03,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:03,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:03,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:06,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:06,285 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:06,285 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:06,290 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:07,209 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:07,209 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:08,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:08,546 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:46:13,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:13,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:13,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:13,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:14,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:14,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:14,654 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:14,659 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:15,217 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:17,220 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:18,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:19,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:19,129 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:19,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:19,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:19,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:21,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:24,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:24,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:24,839 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:24,839 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:24,844 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:25,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:27,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:28,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:28,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:28,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:29,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:29,162 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:29,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:29,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:29,168 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:29,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:31,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:34,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:34,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:34,315 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:34,316 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:34,321 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:35,242 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:35,242 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:38,551 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:46:39,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:39,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:39,660 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:39,661 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:39,666 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:40,245 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:41,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:43,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:43,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:43,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:45,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:48,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:48,143 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:48,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:48,148 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:48,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:49,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:51,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:56,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:57,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:57,204 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:57,204 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:57,209 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:57,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:58,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:58,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:58,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:59,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:01,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:05,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:05,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:05,435 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:05,440 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:06,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:06,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:07,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:08,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:47:10,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:10,102 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:10,102 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:10,106 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:10,274 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:11,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:12,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:13,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:13,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:13,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:14,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:14,424 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:14,424 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:14,429 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:15,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:17,283 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:17,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:20,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:20,453 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:20,453 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:20,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:21,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:22,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:23,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:24,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:24,393 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:24,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:24,398 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:25,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:25,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:28,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:28,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:28,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:28,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:29,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:29,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:29,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:29,526 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:30,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:31,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:33,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:33,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:33,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:33,729 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:33,734 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:34,301 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:35,303 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:38,556 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:47:38,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:38,577 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:38,577 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:38,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:38,583 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:39,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:41,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:42,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:42,678 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:42,679 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:42,684 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:43,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:43,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:43,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:43,311 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:44,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:45,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:47,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:47,407 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:47,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:47,412 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:48,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:49,320 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:50,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:52,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:52,383 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:52,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:52,388 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:53,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:53,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:56,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:58,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:58,115 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:58,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:58,120 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:58,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:58,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:58,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:58,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:59,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:01,462 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:02,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:02,559 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:02,559 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:02,564 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:03,333 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:05,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:06,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:08,554 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:48:08,866 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:08,867 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:08,867 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:08,873 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:09,341 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:11,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:11,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:13,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:13,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:13,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:13,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:13,604 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:13,604 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:13,610 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:14,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:15,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:17,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:18,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:18,342 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:18,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:18,347 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:18,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:19,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:23,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:23,591 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:23,591 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:23,592 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:23,597 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:24,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:25,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:28,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:28,029 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:28,029 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:28,034 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:28,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:28,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:28,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:28,362 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:28,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:29,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:32,852 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:32,853 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:32,853 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:32,858 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:33,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:33,859 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:35,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:38,557 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:48:39,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:41,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:41,110 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:41,111 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:41,116 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:41,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:43,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:43,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:43,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:43,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:45,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:46,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:46,352 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:46,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:46,357 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:46,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:47,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:51,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:52,471 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:52,471 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:52,472 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:52,476 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:53,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:53,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:56,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:58,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:58,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:58,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:58,574 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:58,575 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:58,575 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:58,580 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:59,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:01,402 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:01,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:06,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:06,490 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:06,491 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:06,496 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:07,407 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:07,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:07,496 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:08,559 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:49:12,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:12,093 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:12,093 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:12,098 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:12,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:13,099 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:13,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:13,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:13,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:13,415 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:16,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:16,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:16,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:16,354 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:16,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:17,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:18,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:20,804 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:20,805 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:20,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:20,810 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:21,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:23,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:23,811 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:28,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:28,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:28,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:29,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:29,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:29,281 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:29,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:29,286 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:29,428 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:31,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:33,788 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:33,789 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:33,789 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:33,794 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:34,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:34,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:35,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:38,561 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:49:40,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:40,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:40,534 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:40,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:40,539 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:41,442 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:41,442 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:43,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:43,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:43,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:45,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:45,602 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:45,603 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:45,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:45,608 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:46,445 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:47,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:50,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:51,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:51,037 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:51,038 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:51,043 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:51,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:53,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:56,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:56,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:56,208 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:56,208 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:56,213 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:56,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:57,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:58,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:58,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:58,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:01,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:01,724 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:01,725 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:01,726 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:01,731 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:02,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:03,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:06,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:06,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:06,414 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:06,419 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:06,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:07,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:07,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:08,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:50:12,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:12,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:12,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:12,099 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:12,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:13,099 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:13,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:13,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:13,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:13,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:16,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:16,655 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:16,656 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:16,661 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:17,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:18,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:19,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:23,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:25,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:25,108 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:25,108 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:25,113 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:25,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:27,491 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:28,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:28,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:28,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:29,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:29,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:29,580 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:29,580 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:29,585 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:30,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:31,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:34,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:34,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:34,228 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:34,233 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:34,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:35,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:35,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:38,565 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:50:39,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:39,469 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:39,469 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:39,474 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:39,503 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:40,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:41,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:43,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:43,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:43,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:45,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:45,338 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:45,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:45,343 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:45,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:46,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:47,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:49,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:49,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:49,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:49,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:50,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:51,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:51,860 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:54,499 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:54,500 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:54,500 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:54,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:54,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:55,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:57,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:58,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:58,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:58,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:59,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:59,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:59,414 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:59,419 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:59,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:01,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:03,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:04,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:04,422 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:04,422 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:04,426 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:04,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:05,530 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:08,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:08,571 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:51:09,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:09,523 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:09,523 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:09,528 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:09,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:11,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:13,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:13,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:13,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:14,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:15,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:15,015 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:15,015 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:15,020 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:15,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:17,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:20,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:23,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:23,383 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:23,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:23,388 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:23,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:25,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:25,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:28,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:28,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:28,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:29,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:29,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:29,024 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:29,029 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:29,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:31,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:31,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:33,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:33,451 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:33,451 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:33,456 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:33,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:35,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:36,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:38,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:38,166 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:38,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:38,171 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:38,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:38,570 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:51:39,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:41,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:43,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:43,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:43,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:45,768 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:45,768 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:45,768 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:45,773 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:46,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:46,774 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:47,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:50,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:50,974 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:50,974 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:50,979 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:51,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:51,979 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:53,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:55,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:55,833 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:55,834 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:55,839 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:56,581 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:57,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:57,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:58,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:58,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:58,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:01,804 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:01,805 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:01,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:01,810 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:02,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:03,589 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:03,811 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:08,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:08,160 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:08,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:08,165 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:08,572 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:52:08,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:09,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:09,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:12,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:12,691 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:12,692 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:12,697 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:13,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:13,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:13,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:13,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:13,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:15,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:18,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:18,017 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:18,018 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:18,023 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:18,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:19,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:21,024 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:23,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:23,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:23,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:23,813 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:24,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:25,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:26,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:28,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:28,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:28,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:28,632 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:28,632 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:28,633 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:28,638 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:29,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:29,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:32,639 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:34,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:34,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:34,195 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:34,200 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:34,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:35,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:38,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:38,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:52:42,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:42,365 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:42,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:42,369 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:42,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:43,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:43,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:43,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:43,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:43,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:47,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:47,064 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:47,064 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:47,069 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:47,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:49,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:49,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:51,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:51,388 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:51,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:51,393 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:51,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:53,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:54,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:56,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:56,738 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:56,738 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:56,743 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:57,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:57,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:58,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:58,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:58,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:59,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:02,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:02,180 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:02,180 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:02,185 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:02,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:03,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:05,185 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:06,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:06,461 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:06,461 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:06,466 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:06,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:07,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:08,576 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:53:10,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:12,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:12,691 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:12,692 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:12,696 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:13,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:13,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:13,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:13,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:13,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:16,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:17,527 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:17,527 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:17,527 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:17,533 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:17,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:19,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:21,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:22,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:22,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:22,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:22,247 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:22,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:23,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:26,732 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:26,733 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:26,733 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:26,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:26,738 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:27,672 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:27,672 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:28,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:28,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:28,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:32,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:33,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:33,337 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:33,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:33,342 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:33,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:35,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:37,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:37,800 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:37,801 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:37,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:37,806 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:38,582 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:53:38,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:39,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:43,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:43,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:43,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:43,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:43,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:43,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:43,897 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:43,902 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:44,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:45,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:48,696 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:48,697 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:48,697 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:48,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:48,702 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:49,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:49,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:52,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:52,624 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:52,624 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:52,630 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:52,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:53,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:54,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:56,507 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:56,508 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:56,508 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:56,514 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:56,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:57,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:58,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:58,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:58,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:00,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:01,959 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:01,960 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:01,960 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:01,965 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:02,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:03,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:05,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:08,581 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:54:10,304 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:10,305 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:10,305 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:10,309 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:10,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:11,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:11,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:13,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:13,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:13,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:15,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:15,146 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:15,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:15,151 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:15,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:17,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:17,724 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:20,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:20,342 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:20,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:20,348 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:20,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:21,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:22,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:25,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:25,283 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:25,283 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:25,288 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:25,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:27,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:28,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:28,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:28,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:28,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:29,979 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:29,980 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:29,980 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:29,985 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:30,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:31,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:33,986 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:34,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:34,546 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:34,546 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:34,551 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:34,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:35,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:38,586 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:54:39,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:43,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:43,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:43,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:45,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:46,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:46,081 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:46,081 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:46,086 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:46,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:47,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:50,511 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:50,512 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:50,512 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:50,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:50,518 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:50,754 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:51,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:55,518 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:56,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:56,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:56,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:56,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:56,759 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:57,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:58,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:58,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:58,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:01,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:02,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:02,764 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:02,764 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:02,769 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:03,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:03,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:06,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:08,585 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:55:08,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:08,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:08,976 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:08,981 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:09,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:11,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:11,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:13,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:13,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:13,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:15,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:15,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:15,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:15,345 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:15,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:17,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:17,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:21,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:21,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:21,038 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:21,044 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:21,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:23,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:23,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:25,536 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:25,536 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:25,537 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:25,541 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:25,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:27,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:28,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:28,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:28,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:28,462 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:31,431 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:31,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:31,432 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:31,437 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:31,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:33,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:34,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:36,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:36,190 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:36,190 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:36,195 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:36,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:37,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:38,587 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:55:39,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:41,535 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:41,536 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:41,536 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:41,541 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:41,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:43,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:43,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:43,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:43,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:45,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:46,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:46,832 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:46,833 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:46,837 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:47,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:47,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:50,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:52,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:52,629 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:52,629 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:52,635 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:52,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:53,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:56,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:58,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:58,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:58,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:01,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:01,081 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:01,081 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:01,086 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:01,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:02,086 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:03,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:05,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:05,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:05,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:05,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:05,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:07,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:07,832 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:08,590 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:56:13,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:13,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:13,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:13,454 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:15,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:15,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:15,307 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:15,312 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:15,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:17,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:19,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:20,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:20,638 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:20,638 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:20,643 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:20,844 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:21,847 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:24,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:25,820 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:25,821 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:25,821 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:25,827 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:25,849 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:27,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:28,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:28,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:28,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:30,476 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:30,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:30,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:30,644 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:30,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:30,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:31,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:35,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:36,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:36,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:36,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:36,159 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:36,860 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:37,863 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:38,592 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:56:41,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:43,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:43,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:43,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:44,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:44,582 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:44,582 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:44,587 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:44,867 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:45,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:47,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:49,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:49,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:49,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:49,519 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:49,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:51,878 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:53,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:57,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:57,928 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:57,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:57,933 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:58,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:58,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:58,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:58,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:59,471 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:59,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:02,604 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:02,604 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:02,604 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:02,609 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:02,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:03,889 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:04,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:07,103 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:07,104 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:07,104 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:07,109 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:07,892 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:08,598 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:57:09,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:10,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:13,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:13,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:13,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:13,715 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:13,715 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:13,715 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:13,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:13,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:15,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:15,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:19,266 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:19,267 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:19,267 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:19,272 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:19,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:21,272 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:21,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:25,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:25,204 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:25,204 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:25,209 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:25,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:27,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:27,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:28,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:28,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:28,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:29,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:29,583 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:29,583 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:29,588 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:29,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:31,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:32,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:34,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:34,358 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:34,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:34,364 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:34,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:35,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:38,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:38,596 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:57:38,720 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:38,720 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:38,720 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:38,725 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:38,924 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:39,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:43,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:43,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:43,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:43,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:44,256 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:44,256 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:44,256 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:44,261 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:44,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:45,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:48,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:48,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:48,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:48,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:48,777 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:48,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:49,938 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:53,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:54,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:54,905 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:54,906 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:54,910 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:54,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:55,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:58,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:58,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:58,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:59,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:59,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:59,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:59,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:59,216 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:59,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:01,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:04,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:04,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:04,347 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:04,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:04,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:04,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:05,954 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:08,598 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:58:09,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:09,295 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:09,295 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:09,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:09,300 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:09,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:11,962 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:13,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:13,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:13,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:14,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:14,309 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:14,309 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:14,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:14,314 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:14,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:15,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:18,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:18,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:18,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:18,823 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:18,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:19,823 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:19,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:23,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:23,938 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:23,938 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:23,943 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:23,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:24,944 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:25,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:28,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:28,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:28,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:28,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:28,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:28,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:28,776 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:28,976 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:29,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:30,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:32,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:32,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:32,964 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:32,969 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:32,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:33,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:35,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:37,279 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:37,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:37,280 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:37,285 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:37,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:38,601 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:58:39,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:41,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:42,496 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:42,497 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:42,497 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:42,502 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:42,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:43,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:43,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:43,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:43,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:46,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:46,764 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:46,764 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:46,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:46,769 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:46,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:47,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:51,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:51,496 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:51,496 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:51,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:52,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:52,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:54,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:56,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:56,253 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:56,253 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:56,258 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:57,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:58,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:58,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:58,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:58,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:58,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:00,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:00,363 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:00,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:00,368 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:01,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:02,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:03,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:04,476 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:04,477 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:04,477 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:04,483 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:05,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:06,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:08,483 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:08,603 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:59:11,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:11,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:11,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:11,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:12,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:13,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:13,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:13,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:14,029 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:14,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:15,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:15,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:15,878 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:15,885 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:16,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:18,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:19,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:20,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:20,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:20,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:20,155 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:21,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:22,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:25,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:25,141 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:25,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:25,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:25,146 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:26,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:26,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:28,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:28,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:28,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:29,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:29,652 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:29,652 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:29,657 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:30,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:30,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:32,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:33,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:33,955 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:33,955 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:33,960 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:34,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:35,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:36,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:38,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:38,069 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:38,070 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:38,077 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:38,605 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:59:39,061 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:40,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:41,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:43,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:43,003 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:43,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:43,008 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:43,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:43,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:43,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:43,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:44,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:47,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:47,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:47,873 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:47,873 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:47,878 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:48,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:50,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:52,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:55,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:55,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:55,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:55,776 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:56,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:58,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:58,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:58,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:58,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:58,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:01,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:01,376 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:01,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:01,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:02,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:04,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:04,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:06,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:06,218 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:06,218 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:06,223 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:07,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:08,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:08,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:00:09,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:12,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:12,587 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:12,588 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:12,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:13,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:13,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:13,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:13,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:14,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:15,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:17,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:17,586 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:17,586 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:17,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:18,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:20,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:20,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:22,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:22,377 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:22,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:22,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:23,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:24,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:26,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:27,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:27,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:27,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:27,011 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:27,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:28,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:28,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:28,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:28,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:31,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:33,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:33,339 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:33,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:33,344 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:34,123 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:36,128 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:37,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:38,613 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:00:39,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:39,206 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:39,207 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:39,212 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:40,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:40,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:43,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:43,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:43,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:43,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:43,706 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:43,706 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:43,706 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:43,711 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:44,137 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:46,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:48,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:50,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:50,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:50,286 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:50,291 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:51,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:52,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:54,292 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:58,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:58,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:58,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:58,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:58,570 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:58,570 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:58,575 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:59,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:59,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:00,162 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:03,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:03,450 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:03,450 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:03,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:04,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:05,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:06,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:07,752 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:07,753 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:07,753 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:07,758 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:08,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:08,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:01:10,178 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:10,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:12,543 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:12,544 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:12,544 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:12,549 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:13,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:13,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:13,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:13,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:14,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:16,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:17,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:17,610 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:17,610 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:17,615 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:18,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:20,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:21,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:22,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:22,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:22,787 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:22,792 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:23,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:24,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:26,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:27,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:27,300 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:27,301 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:27,306 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:28,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:28,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:28,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:28,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:28,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:32,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:32,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:32,758 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:32,758 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:32,763 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:33,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:34,211 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:37,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:38,621 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:01:38,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:38,688 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:38,688 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:38,693 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:39,217 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:40,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:43,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:43,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:43,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:43,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:43,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:43,431 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:43,431 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:43,437 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:44,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:46,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:47,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:47,971 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:47,971 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:47,976 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:48,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:48,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:50,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:51,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:51,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:51,778 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:51,783 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:52,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:54,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:54,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:56,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:56,060 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:56,060 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:56,065 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:56,242 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:58,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:58,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:58,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:58,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:00,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:01,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:01,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:01,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:01,338 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:02,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:02,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:06,052 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:06,053 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:06,053 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:06,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:06,059 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:06,256 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:08,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:08,617 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:02:10,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:10,304 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:10,304 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:10,309 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:11,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:11,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:12,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:13,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:13,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:13,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:15,812 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:15,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:15,813 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:15,818 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:16,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:16,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:18,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:21,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:22,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:22,552 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:22,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:22,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:23,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:24,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:27,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:28,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:28,160 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:28,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:28,165 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:28,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:28,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:28,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:28,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:30,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:33,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:33,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:33,974 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:33,975 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:33,980 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:34,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:36,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:38,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:02:38,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:39,596 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:39,597 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:39,597 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:39,602 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:40,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:42,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:43,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:43,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:43,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:44,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:44,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:44,438 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:44,439 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:44,444 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:45,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:46,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:48,808 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:48,809 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:48,809 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:48,814 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:49,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:49,815 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:50,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:53,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:53,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:53,475 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:53,480 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:54,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:55,480 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:56,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:58,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:58,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:58,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:01,462 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:02,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:02,039 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:02,040 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:02,044 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:02,338 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:04,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:07,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:07,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:07,535 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:07,535 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:07,540 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:08,346 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:08,622 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:03:10,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:12,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:12,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:12,195 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:12,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:12,201 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:12,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:13,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:13,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:13,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:14,359 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:17,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:17,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:17,802 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:17,802 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:17,807 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:18,362 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:20,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:22,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:24,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:24,271 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:24,271 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:24,276 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:24,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:26,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:28,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:28,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:28,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:28,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:29,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:29,193 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:29,193 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:29,199 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:29,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:30,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:34,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:35,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:35,974 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:35,974 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:35,980 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:36,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:38,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:38,624 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:03:39,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:42,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:42,006 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:42,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:42,011 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:42,396 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:43,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:43,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:43,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:44,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:45,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:46,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:46,993 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:46,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:46,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:47,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:48,407 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:51,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:53,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:53,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:53,013 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:53,018 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:53,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:54,415 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:57,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:58,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:58,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:58,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:01,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:01,418 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:01,418 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:01,424 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:01,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:02,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:02,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:05,799 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:05,800 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:05,800 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:05,806 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:06,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:07,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:08,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:08,629 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:04:11,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:11,005 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:11,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:11,010 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:11,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:12,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:13,011 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:13,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:13,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:13,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:18,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:18,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:18,481 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:18,481 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:18,487 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:19,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:20,454 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:23,526 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:25,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:25,200 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:25,200 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:25,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:25,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:26,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:28,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:28,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:28,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:29,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:31,543 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:31,544 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:31,544 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:31,549 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:32,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:34,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:34,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:38,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:38,344 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:38,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:38,349 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:38,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:38,628 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:04:39,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:40,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:42,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:42,409 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:42,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:42,415 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:42,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:43,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:43,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:43,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:44,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:45,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:46,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:46,873 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:46,873 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:46,878 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:47,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:48,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:50,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:51,947 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:51,948 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:51,948 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:51,954 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:52,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:54,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:55,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:56,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:56,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:56,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:56,454 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:56,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:58,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:58,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:58,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:58,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:01,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:02,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:02,145 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:02,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:02,151 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:02,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:04,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:06,831 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:06,832 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:06,832 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:06,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:06,838 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:07,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:08,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:08,633 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:05:11,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:11,226 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:11,227 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:11,232 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:11,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:12,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:12,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:13,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:13,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:13,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:17,460 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:18,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:18,215 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:18,215 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:18,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:18,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:20,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:23,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:27,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:27,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:27,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:27,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:27,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:28,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:28,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:28,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:28,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:28,554 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:31,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:31,295 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:31,296 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:31,301 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:31,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:32,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:34,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:35,531 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:35,531 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:35,532 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:35,537 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:35,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:36,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:38,632 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:05:39,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:40,550 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:40,551 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:40,551 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:40,556 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:40,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:42,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:43,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:43,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:43,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:45,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:46,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:46,353 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:46,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:46,358 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:46,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:48,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:50,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:50,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:50,872 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:50,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:50,878 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:51,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:52,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:55,879 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:56,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:56,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:56,637 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:56,642 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:57,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:58,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:58,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:58,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:58,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:01,443 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:02,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:02,021 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:02,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:02,027 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:02,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:04,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:06,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:06,835 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:06,836 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:06,836 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:06,841 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:07,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:08,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:08,634 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:06:11,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:11,326 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:11,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:11,331 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:11,617 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:12,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:12,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:13,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:13,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:13,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:16,431 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:16,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:16,432 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:16,437 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:16,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:17,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:18,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:22,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:22,015 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:22,015 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:22,020 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:22,630 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:23,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:24,636 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:26,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:26,542 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:26,542 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:26,547 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:26,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:28,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:28,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:28,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:28,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:28,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:33,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:33,427 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:33,427 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:33,432 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:33,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:33,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:34,650 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:37,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:37,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:37,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:37,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:38,637 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:06:38,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:38,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:40,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:42,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:42,391 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:42,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:42,396 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:42,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:43,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:43,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:43,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:44,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:44,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:47,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:47,909 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:47,910 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:47,915 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:48,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:49,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:50,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:52,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:52,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:52,534 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:52,539 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:52,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:54,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:55,539 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:57,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:57,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:57,224 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:57,228 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:57,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:58,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:58,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:58,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:58,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:01,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:04,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:04,006 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:04,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:04,011 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:04,692 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:06,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:07,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:08,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:08,431 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:08,431 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:08,436 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:08,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:07:08,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:10,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:12,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:12,479 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:12,479 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:12,479 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:12,485 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:12,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:13,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:13,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:13,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:14,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:17,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:17,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:17,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:17,338 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:17,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:18,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:18,715 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:23,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:23,820 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:23,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:23,821 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:23,826 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:24,722 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:26,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:28,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:28,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:28,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:28,132 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:28,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:28,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:28,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:28,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:28,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:30,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:33,104 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:33,104 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:33,105 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:33,110 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:33,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:34,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:34,738 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:37,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:37,423 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:37,423 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:37,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:37,741 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:38,645 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:07:38,743 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:39,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:42,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:42,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:42,246 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:42,253 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:42,747 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:43,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:43,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:43,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:44,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:45,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:48,859 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:48,860 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:48,860 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:48,865 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:49,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:50,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:50,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:53,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:53,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:53,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:53,339 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:53,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:54,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:56,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:58,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:58,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:58,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:59,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:59,670 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:59,671 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:59,675 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:59,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:00,776 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:01,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:04,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:04,586 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:04,586 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:04,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:04,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:06,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:07,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:08,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:08:09,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:09,086 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:09,086 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:09,091 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:09,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:10,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:13,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:13,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:13,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:13,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:13,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:13,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:13,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:13,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:13,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:14,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:18,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:21,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:21,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:21,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:21,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:21,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:22,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:23,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:25,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:25,467 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:25,467 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:25,472 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:25,811 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:26,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:28,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:28,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:28,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:29,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:32,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:32,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:32,286 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:32,291 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:32,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:34,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:35,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:37,031 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:37,032 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:37,032 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:37,037 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:37,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:38,649 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:08:38,831 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:40,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:42,468 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:42,468 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:42,469 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:42,474 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:42,834 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:43,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:43,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:43,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:44,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:46,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:49,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:49,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:49,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:49,243 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:49,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:50,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:52,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:53,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:53,711 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:53,712 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:53,717 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:53,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:54,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:57,718 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:58,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:58,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:58,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:58,827 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:58,828 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:58,828 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:58,833 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:58,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:00,863 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:02,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:03,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:03,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:03,286 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:03,291 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:03,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:04,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:08,292 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:08,648 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:09:12,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:12,427 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:12,427 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:12,432 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:12,879 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:13,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:13,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:13,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:13,443 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:14,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:18,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:18,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:18,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:18,162 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:18,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:19,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:20,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:23,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:23,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:23,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:23,177 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:23,896 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:24,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:25,178 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:28,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:28,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:28,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:30,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:30,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:30,390 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:30,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:30,396 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:30,905 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:32,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:35,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:35,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:35,882 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:35,882 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:35,888 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:35,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:36,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:38,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:09:40,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:41,047 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:41,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:41,048 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:41,053 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:41,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:42,924 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:43,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:43,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:43,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:45,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:45,873 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:45,873 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:45,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:45,879 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:45,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:48,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:50,918 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:51,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:51,084 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:51,085 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:51,089 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:51,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:52,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:56,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:56,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:56,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:56,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:56,813 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:56,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:58,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:58,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:58,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:58,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:01,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:02,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:02,268 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:02,268 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:02,273 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:02,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:04,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:07,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:08,652 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:10:10,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:10,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:10,790 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:10,795 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:10,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:12,796 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:12,967 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:13,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:13,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:13,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:15,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:15,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:15,654 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:15,659 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:15,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:16,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:18,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:21,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:21,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:21,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:21,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:21,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:22,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:24,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:25,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:25,522 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:25,522 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:25,528 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:25,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:26,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:28,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:28,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:28,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:29,526 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:30,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:30,237 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:30,237 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:30,243 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:30,988 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:32,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:35,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:35,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:35,694 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:35,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:35,701 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:35,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:36,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:38,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:10:40,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:42,639 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:42,640 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:42,641 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:42,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:43,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:43,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:43,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:43,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:45,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:46,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:49,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:49,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:49,612 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:49,617 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:50,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:51,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:51,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:55,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:55,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:55,550 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:55,556 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:56,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:57,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:57,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:58,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:58,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:58,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:00,787 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:00,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:00,788 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:00,793 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:01,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:02,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:03,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:05,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:05,368 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:05,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:05,380 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:06,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:07,025 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:08,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:08,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:11:09,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:09,794 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:09,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:09,800 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:10,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:11,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:13,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:13,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:13,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:13,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:15,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:15,786 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:15,787 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:15,791 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:16,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:17,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:18,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:21,528 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:21,529 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:21,529 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:21,534 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:22,040 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:23,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:24,535 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:26,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:26,311 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:26,311 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:26,316 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:27,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:28,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:28,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:28,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:29,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:29,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:30,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:30,857 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:30,857 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:30,862 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:31,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:33,053 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:34,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:36,787 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:36,788 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:36,788 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:36,793 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:37,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:38,658 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:11:39,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:40,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:43,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:43,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:43,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:43,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:43,589 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:43,589 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:43,594 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:44,061 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:45,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:46,595 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:48,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:48,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:48,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:48,183 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:49,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:51,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:52,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:52,867 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:52,868 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:52,868 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:52,873 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:53,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:55,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:56,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:56,986 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:56,986 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:56,991 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:57,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:57,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:58,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:58,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:58,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:59,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:01,891 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:01,892 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:01,892 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:01,897 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:02,081 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:03,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:03,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:06,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:06,216 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:06,216 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:06,221 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:07,086 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:08,663 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:12:09,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:09,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:10,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:10,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:10,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:10,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:11,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:13,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:13,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:13,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:13,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:15,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:15,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:15,680 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:15,680 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:15,686 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:16,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:17,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:19,983 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:19,984 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:19,985 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:19,990 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:20,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:20,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:21,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:24,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:24,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:24,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:24,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:25,105 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:26,446 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:27,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:28,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:28,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:28,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:29,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:29,954 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:29,954 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:29,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:30,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:31,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:31,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:36,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:36,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:36,039 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:36,044 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:36,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:37,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:37,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:38,662 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:12:40,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:40,533 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:40,534 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:40,539 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:41,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:42,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:43,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:43,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:43,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:43,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:46,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:46,956 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:46,956 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:46,962 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:47,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:47,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:49,129 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:51,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:51,488 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:51,488 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:51,493 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:52,131 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:53,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:53,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:58,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:58,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:58,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:58,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:00,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:00,110 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:00,110 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:00,116 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:00,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:03,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:04,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:05,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:05,873 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:05,873 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:05,879 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:06,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:07,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:08,665 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:13:09,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:10,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:10,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:10,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:10,819 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:11,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:13,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:13,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:13,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:13,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:15,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:15,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:15,683 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:15,683 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:15,688 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:16,155 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:17,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:20,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:22,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:22,051 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:22,051 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:22,057 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:22,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:23,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:26,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:26,604 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:26,609 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:26,609 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:26,614 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:27,167 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:28,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:28,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:29,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:31,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:35,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:35,028 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:35,029 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:35,034 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:35,173 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:37,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:37,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:38,667 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:13:39,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:39,291 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:39,292 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:39,296 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:40,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:41,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:42,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:43,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:43,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:43,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:44,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:44,008 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:44,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:44,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:44,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:45,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:48,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:48,692 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:48,693 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:48,693 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:48,698 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:49,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:51,192 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:53,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:53,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:53,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:53,178 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:53,183 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:53,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:55,196 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:58,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:58,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:58,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:58,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:59,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:59,098 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:59,098 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:59,103 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:59,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:01,202 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:04,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:05,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:05,251 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:05,251 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:05,256 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:06,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:07,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:08,669 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:14:09,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:11,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:11,367 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:11,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:11,373 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:12,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:13,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:13,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:13,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:13,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:15,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:18,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:18,584 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:18,585 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:18,589 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:19,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:21,220 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:21,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:23,875 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:23,876 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:23,876 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:23,880 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:24,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:25,225 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:26,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:28,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:28,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:28,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:29,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:29,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:29,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:29,321 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:30,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:31,231 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:32,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:35,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:35,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:35,498 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:35,503 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:36,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:37,237 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:37,503 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:38,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:14:40,026 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:40,026 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:40,026 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:40,031 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:40,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:41,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:43,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:43,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:43,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:43,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:44,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:44,306 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:44,306 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:44,311 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:45,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:47,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:48,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:48,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:48,824 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:48,825 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:48,830 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:49,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:51,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:53,830 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:53,927 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:53,927 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:53,927 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:53,933 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:54,254 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:55,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:58,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:58,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:58,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:58,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:58,955 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:58,955 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:58,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:58,960 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:59,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:01,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:03,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:04,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:04,704 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:04,705 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:04,709 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:05,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:07,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:08,673 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:15:09,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:10,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:10,963 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:10,963 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:10,968 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:11,272 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:13,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:13,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:13,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:13,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:15,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:15,437 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:15,437 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:15,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:15,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:16,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:17,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:20,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:20,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:20,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:20,247 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:20,283 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:21,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:21,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:25,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:25,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:25,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:25,780 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:26,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:26,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:27,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:28,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:28,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:30,454 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:30,454 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:30,454 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:30,459 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:31,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:32,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:33,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:35,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:35,638 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:35,638 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:35,643 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:36,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:37,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:37,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:38,679 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:15:41,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:41,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:41,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:41,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:41,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:43,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:43,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:43,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:43,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:45,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:45,856 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:45,857 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:45,861 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:46,307 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:47,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:48,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:50,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:50,711 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:50,712 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:50,716 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:51,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:53,316 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:54,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:55,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:55,235 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:55,235 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:55,240 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:55,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:57,320 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:58,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:58,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:00,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:00,374 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:00,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:00,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:00,379 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:01,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:03,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:04,866 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:04,867 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:04,867 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:04,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:05,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:05,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:07,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:08,678 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:16:09,527 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:09,528 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:09,528 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:09,533 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:10,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:11,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:11,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:13,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:13,327 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:13,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:14,668 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:14,668 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:14,668 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:14,673 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:15,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:16,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:17,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:21,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:23,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:23,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:23,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:23,340 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:23,344 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:25,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:27,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:28,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:28,327 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:28,797 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:28,797 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:28,797 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:28,802 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:29,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:31,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:32,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:34,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:34,019 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:34,020 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:34,024 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:34,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:35,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:38,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:38,680 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:16:40,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:40,561 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:40,561 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:40,567 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:41,362 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:43,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:43,327 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:43,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:43,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:45,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:45,692 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:45,692 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:45,697 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:46,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:47,368 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:48,736 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:50,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:50,477 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:50,477 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:50,482 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:51,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:53,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:54,483 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:57,099 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:57,100 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:57,100 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:57,105 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:57,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:58,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:58,327 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:59,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:59,595 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:01,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:01,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:01,883 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:01,889 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:02,382 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:03,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:04,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:08,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:08,052 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:08,052 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:08,057 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:08,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:08,682 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:17:09,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:10,683 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:13,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:13,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:13,227 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:13,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:13,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:13,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:13,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:13,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:15,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:16,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:18,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:18,042 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:18,042 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:18,047 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:18,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:19,402 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:22,048 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:22,825 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:22,826 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:22,826 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:22,832 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:23,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:25,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:27,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:28,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:28,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:28,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:28,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:28,632 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:28,632 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:28,637 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:29,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:31,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:33,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:34,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:34,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:34,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:34,527 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:35,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:37,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:38,684 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:17:38,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:39,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:39,367 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:39,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:39,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:39,420 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:41,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:43,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:43,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:44,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:44,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:44,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:44,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:44,244 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:44,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:45,428 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:49,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:50,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:50,446 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:50,446 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:50,451 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:51,435 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:51,435 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:54,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:56,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:56,360 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:56,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:56,365 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:56,438 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:57,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:58,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:58,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:59,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:01,866 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:01,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:01,867 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:01,871 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:02,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:03,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:04,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:06,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:06,372 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:06,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:06,377 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:06,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:07,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:08,689 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:18:10,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:12,060 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:12,061 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:12,061 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:12,066 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:12,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:13,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:13,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:13,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:13,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:16,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:16,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:16,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:16,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:16,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:17,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:19,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:20,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:20,399 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:20,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:20,404 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:20,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:21,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:22,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:25,864 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:25,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:25,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:25,869 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:26,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:27,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:27,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:28,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:28,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:33,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:33,098 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:33,098 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:33,098 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:33,103 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:33,472 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:35,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:37,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:37,652 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:37,652 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:37,657 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:38,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:38,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:38,689 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:18:39,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:41,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:41,756 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:41,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:41,761 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:42,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:43,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:43,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:43,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:44,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:45,796 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:45,797 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:45,797 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:45,802 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:46,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:47,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:49,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:51,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:51,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:51,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:51,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:51,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:53,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:55,216 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:55,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:55,462 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:55,462 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:55,467 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:55,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:57,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:58,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:58,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:00,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:05,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:05,019 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:05,019 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:05,024 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:05,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:06,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:07,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:08,694 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:19:10,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:10,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:10,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:10,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:10,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:11,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:11,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:13,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:13,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:13,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:14,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:14,492 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:14,492 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:14,497 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:14,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:15,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:16,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:19,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:19,644 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:19,644 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:19,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:20,521 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:21,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:21,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:24,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:24,158 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:24,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:24,163 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:24,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:25,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:27,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:28,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:28,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:28,632 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:28,632 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:28,637 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:29,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:31,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:32,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:33,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:33,334 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:33,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:33,339 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:33,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:35,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:38,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:38,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:38,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:38,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:38,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:38,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:38,693 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:19:39,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:43,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:43,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:43,299 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:43,303 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:43,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:43,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:43,545 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:43,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:45,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:48,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:49,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:49,070 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:49,070 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:49,075 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:49,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:51,554 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:53,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:53,765 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:53,765 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:53,765 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:53,770 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:54,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:55,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:58,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:59,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:00,768 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:00,769 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:00,769 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:00,774 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:01,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:03,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:04,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:05,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:05,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:05,239 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:05,243 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:05,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:07,567 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:08,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:20:09,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:09,675 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:09,675 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:09,681 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:10,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:10,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:11,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:13,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:13,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:13,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:16,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:18,185 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:18,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:18,186 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:18,192 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:18,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:19,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:22,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:23,262 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:23,263 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:23,263 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:23,268 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:23,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:25,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:27,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:28,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:28,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:31,460 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:31,461 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:31,461 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:31,466 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:31,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:32,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:33,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:36,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:36,145 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:36,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:36,150 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:36,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:37,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:38,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:38,697 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:20:42,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:42,060 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:42,060 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:42,066 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:42,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:43,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:43,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:43,600 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:46,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:46,090 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:46,090 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:46,095 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:46,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:47,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:49,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:51,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:51,711 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:51,711 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:51,716 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:52,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:53,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:54,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:57,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:57,303 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:57,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:57,308 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:57,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:58,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:59,617 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:00,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:01,688 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:01,689 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:01,689 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:01,694 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:02,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:03,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:05,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:08,700 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:21:10,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:10,163 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:10,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:10,168 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:10,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:11,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:11,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:13,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:13,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:13,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:16,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:16,613 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:16,613 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:16,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:16,618 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:16,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:17,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:21,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:21,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:21,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:21,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:21,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:22,320 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:23,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:27,320 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:27,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:27,378 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:27,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:27,384 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:27,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:28,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:28,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:29,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:32,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:35,855 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:35,855 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:35,855 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:35,860 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:36,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:37,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:37,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:38,705 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:21:41,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:41,361 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:41,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:41,367 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:41,655 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:43,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:43,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:43,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:43,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:46,752 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:46,753 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:46,753 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:46,758 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:47,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:48,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:49,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:52,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:52,191 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:52,191 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:52,196 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:52,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:53,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:54,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:57,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:57,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:57,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:57,318 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:57,669 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:58,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:58,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:59,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:59,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:02,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:02,793 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:02,793 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:02,798 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:03,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:04,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:05,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:07,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:07,319 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:07,319 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:07,324 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:07,677 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:08,704 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:22:09,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:10,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:11,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:11,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:11,639 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:11,644 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:11,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:13,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:13,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:13,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:13,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:16,104 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:16,105 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:16,105 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:16,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:16,110 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:16,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:17,692 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:21,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:22,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:22,245 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:22,245 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:22,251 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:22,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:23,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:26,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:26,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:26,354 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:26,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:26,360 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:26,699 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:27,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:28,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:28,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:28,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:30,904 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:30,905 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:30,905 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:30,910 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:31,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:31,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:33,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:36,283 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:36,284 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:36,284 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:36,289 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:36,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:37,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:37,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:38,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:22:41,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:41,282 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:41,282 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:41,287 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:41,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:43,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:43,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:43,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:43,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:43,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:47,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:47,002 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:47,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:47,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:47,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:49,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:49,722 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:54,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:54,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:54,173 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:54,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:54,178 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:54,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:55,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:58,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:58,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:58,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:58,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:58,981 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:58,981 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:58,986 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:59,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:00,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:01,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:04,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:04,397 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:04,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:04,402 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:04,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:05,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:05,738 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:08,708 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:23:09,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:09,786 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:09,786 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:09,791 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:10,741 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:10,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:11,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:13,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:13,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:13,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:16,546 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:17,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:17,041 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:17,042 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:17,046 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:17,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:19,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:22,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:22,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:22,472 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:22,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:22,478 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:22,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:23,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:27,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:28,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:28,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:28,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:30,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:30,824 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:30,825 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:30,829 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:31,763 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:32,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:33,766 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:37,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:37,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:37,246 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:37,251 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:37,769 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:38,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:38,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:23:39,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:43,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:43,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:43,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:43,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:43,584 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:43,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:43,585 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:43,590 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:43,775 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:45,778 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:48,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:48,031 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:48,031 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:48,036 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:48,780 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:49,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:49,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:52,852 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:52,852 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:52,853 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:52,857 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:53,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:54,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:55,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:57,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:57,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:57,436 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:57,441 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:57,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:58,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:58,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:58,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:59,796 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:00,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:02,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:02,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:02,675 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:02,680 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:02,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:03,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:05,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:07,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:07,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:07,195 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:07,200 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:07,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:08,713 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:24:09,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:10,714 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:11,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:11,415 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:11,416 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:11,420 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:11,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:13,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:13,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:13,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:13,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:15,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:15,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:15,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:15,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:15,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:16,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:17,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:21,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:24,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:24,177 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:24,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:24,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:24,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:25,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:27,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:28,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:28,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:28,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:28,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:28,697 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:28,697 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:28,702 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:28,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:29,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:32,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:33,299 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:33,300 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:33,300 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:33,305 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:33,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:35,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:37,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:37,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:37,550 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:37,555 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:37,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:38,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:38,715 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:24:39,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:42,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:42,017 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:42,017 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:42,022 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:42,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:43,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:43,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:43,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:43,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:43,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:48,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:48,111 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:48,111 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:48,116 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:48,847 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:49,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:49,850 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:52,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:52,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:52,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:52,526 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:52,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:53,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:54,527 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:58,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:58,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:58,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:59,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:01,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:01,615 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:01,615 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:01,621 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:01,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:03,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:04,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:06,056 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:06,057 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:06,057 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:06,062 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:06,864 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:07,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:08,721 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:25:09,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:10,193 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:10,194 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:10,194 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:10,199 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:10,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:11,872 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:13,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:13,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:13,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:14,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:14,517 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:14,518 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:14,522 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:14,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:15,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:15,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:20,562 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:22,546 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:22,547 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:22,547 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:22,552 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:22,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:23,884 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:26,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:28,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:28,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:28,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:31,119 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:31,120 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:31,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:31,125 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:31,889 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:32,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:33,892 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:35,550 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:35,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:35,551 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:35,556 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:35,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:37,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:37,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:38,719 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:25:40,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:40,653 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:40,653 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:40,658 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:40,896 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:41,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:42,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:43,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:43,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:43,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:44,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:44,917 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:44,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:44,922 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:45,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:45,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:47,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:49,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:49,373 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:49,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:49,378 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:49,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:51,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:53,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:56,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:56,356 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:56,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:56,362 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:56,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:57,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:58,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:58,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:58,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:58,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:01,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:01,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:01,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:01,014 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:01,917 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:03,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:04,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:05,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:05,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:05,494 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:05,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:05,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:07,925 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:08,724 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:26:09,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:10,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:10,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:10,878 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:10,883 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:10,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:11,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:13,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:13,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:13,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:15,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:16,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:16,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:16,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:16,059 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:16,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:17,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:20,515 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:20,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:20,516 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:20,521 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:20,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:21,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:21,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:26,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:28,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:28,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:28,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:28,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:28,861 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:28,862 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:28,867 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:28,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:29,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:31,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:33,272 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:33,272 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:33,273 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:33,279 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:33,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:35,954 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:37,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:38,724 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:26:39,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:39,839 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:39,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:39,845 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:39,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:41,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:42,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:43,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:43,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:43,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:44,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:44,635 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:44,635 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:44,640 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:44,961 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:45,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:48,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:49,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:49,281 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:49,281 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:49,286 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:49,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:51,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:54,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:54,654 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:54,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:54,655 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:54,660 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:54,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:55,974 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:58,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:58,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:58,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:59,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:00,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:00,132 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:00,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:00,137 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:00,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:01,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:05,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:08,672 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:08,673 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:08,673 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:08,678 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:08,726 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:27:08,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:09,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:10,728 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:13,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:13,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:13,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:14,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:14,900 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:14,900 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:14,905 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:14,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:15,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:15,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:19,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:19,360 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:19,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:19,365 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:19,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:21,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:21,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:24,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:24,742 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:24,743 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:24,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:24,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:26,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:26,748 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:28,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:28,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:28,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:29,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:29,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:29,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:29,312 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:30,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:32,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:32,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:33,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:33,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:33,439 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:33,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:34,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:36,010 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:37,446 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:38,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:27:39,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:39,037 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:39,037 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:39,042 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:40,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:40,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:42,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:42,910 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:42,911 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:42,911 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:42,916 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:43,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:43,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:43,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:43,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:44,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:48,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:49,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:49,691 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:49,691 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:49,696 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:50,022 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:52,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:53,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:55,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:55,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:55,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:55,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:56,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:58,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:58,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:58,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:58,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:59,538 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:02,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:02,635 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:02,635 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:02,640 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:03,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:04,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:04,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:08,730 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:28:08,847 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:08,848 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:08,848 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:08,853 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:09,040 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:09,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:10,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:13,119 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:13,120 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:13,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:13,125 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:13,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:13,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:13,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:14,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:14,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:15,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:17,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:17,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:17,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:17,162 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:18,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:20,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:21,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:22,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:22,587 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:22,587 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:22,592 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:23,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:24,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:26,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:27,932 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:27,932 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:27,933 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:27,938 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:28,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:28,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:28,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:28,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:30,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:32,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:33,783 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:33,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:33,784 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:33,788 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:34,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:36,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:37,826 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:38,736 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:28:42,230 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:42,231 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:42,231 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:42,236 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:43,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:43,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:43,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:43,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:43,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:44,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:48,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:50,531 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:50,531 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:50,532 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:50,537 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:51,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:52,081 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:54,574 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:56,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:56,626 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:56,626 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:56,631 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:57,084 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:58,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:58,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:58,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:58,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:59,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:01,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:01,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:01,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:01,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:02,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:04,092 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:05,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:05,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:05,428 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:05,428 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:05,434 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:06,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:08,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:08,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:29:09,511 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:09,512 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:09,513 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:09,518 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:10,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:10,519 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:12,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:13,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:13,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:13,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:13,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:13,794 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:13,794 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:13,799 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:14,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:15,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:16,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:20,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:22,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:22,259 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:22,259 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:22,264 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:23,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:24,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:26,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:26,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:26,522 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:26,522 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:26,527 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:27,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:28,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:28,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:28,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:28,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:31,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:34,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:34,670 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:34,670 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:34,675 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:35,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:36,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:36,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:38,740 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:29:39,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:39,128 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:39,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:39,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:40,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:40,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:42,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:43,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:43,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:43,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:47,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:48,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:48,062 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:48,062 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:48,068 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:48,137 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:50,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:52,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:52,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:52,458 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:52,463 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:53,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:53,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:54,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:56,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:56,859 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:56,859 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:56,864 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:57,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:58,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:58,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:58,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:58,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:58,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:01,746 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:01,747 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:01,747 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:01,752 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:02,155 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:03,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:04,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:06,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:06,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:06,608 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:06,613 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:07,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:08,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:08,739 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:30:09,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:11,176 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:11,177 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:11,177 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:11,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:12,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:12,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:13,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:13,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:13,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:15,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:19,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:19,377 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:19,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:19,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:20,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:21,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:22,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:25,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:25,591 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:25,591 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:25,596 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:26,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:26,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:28,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:28,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:28,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:28,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:29,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:29,811 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:29,811 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:29,816 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:30,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:31,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:32,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:34,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:34,862 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:34,862 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:34,867 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:35,185 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:36,187 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:36,868 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:38,741 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:30:39,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:39,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:39,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:39,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:39,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:40,192 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:42,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:43,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:43,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:43,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:43,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:43,586 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:43,586 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:43,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:44,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:46,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:47,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:47,934 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:47,935 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:47,935 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:47,940 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:48,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:50,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:52,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:52,152 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:52,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:52,157 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:52,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:53,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:54,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:58,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:58,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:58,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:58,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:00,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:00,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:00,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:00,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:01,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:02,211 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:03,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:05,272 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:05,276 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:05,277 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:05,282 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:06,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:06,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:08,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:31:08,745 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:10,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:10,004 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:10,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:10,010 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:10,217 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:12,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:13,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:13,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:13,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:14,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:14,302 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:14,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:14,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:14,308 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:15,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:16,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:18,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:18,581 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:18,581 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:18,586 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:19,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:19,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:20,231 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:23,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:23,083 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:23,084 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:23,089 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:23,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:24,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:25,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:28,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:28,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:28,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:28,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:28,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:28,639 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:28,644 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:29,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:30,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:30,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:33,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:33,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:33,788 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:33,793 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:34,242 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:35,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:36,245 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:38,746 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:31:38,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:38,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:38,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:38,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:39,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:40,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:40,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:43,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:43,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:43,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:44,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:44,339 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:44,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:44,344 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:45,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:46,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:46,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:49,062 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:49,063 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:49,063 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:49,068 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:49,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:50,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:52,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:53,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:53,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:53,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:53,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:54,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:56,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:57,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:58,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:58,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:58,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:02,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:02,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:02,058 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:02,063 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:02,267 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:03,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:04,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:08,102 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:08,751 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:32:10,086 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:10,087 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:10,088 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:10,093 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:10,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:12,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:13,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:13,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:13,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:13,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:15,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:15,067 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:15,067 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:15,072 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:15,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:16,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:19,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:19,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:19,433 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:19,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:19,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:20,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:22,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:24,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:25,220 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:25,220 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:25,221 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:25,226 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:25,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:26,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:28,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:28,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:28,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:29,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:29,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:29,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:29,889 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:29,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:30,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:32,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:34,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:34,646 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:34,646 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:34,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:34,651 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:35,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:36,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:38,751 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:32:38,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:38,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:38,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:38,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:39,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:39,920 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:40,308 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:43,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:43,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:43,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:44,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:44,083 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:44,084 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:44,089 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:44,311 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:45,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:46,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:49,244 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:49,245 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:49,245 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:49,250 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:49,316 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:50,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:50,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:53,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:53,690 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:53,690 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:53,695 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:54,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:55,696 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:56,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:58,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:58,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:58,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:59,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:59,795 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:59,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:59,800 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:00,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:00,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:02,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:05,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:05,021 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:05,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:05,025 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:05,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:06,026 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:06,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:08,756 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:33:09,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:09,385 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:09,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:09,390 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:10,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:10,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:11,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:13,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:13,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:13,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:16,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:16,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:16,847 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:16,847 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:16,852 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:17,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:18,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:21,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:23,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:23,344 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:23,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:23,349 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:23,349 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:24,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:27,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:28,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:28,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:28,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:31,850 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:31,851 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:31,851 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:31,856 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:32,357 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:32,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:34,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:37,894 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:38,755 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:33:40,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:40,348 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:40,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:40,353 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:40,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:42,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:43,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:43,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:43,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:43,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:46,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:46,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:46,448 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:46,452 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:47,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:48,373 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:49,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:51,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:51,143 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:51,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:51,148 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:51,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:52,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:55,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:56,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:56,442 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:56,443 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:56,448 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:57,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:58,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:58,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:58,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:58,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:00,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:01,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:01,303 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:01,304 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:01,309 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:01,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:02,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:06,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:07,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:07,593 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:07,593 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:07,598 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:08,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:08,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:34:10,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:11,759 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:12,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:12,662 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:12,663 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:12,667 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:13,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:13,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:13,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:13,396 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:14,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:17,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:20,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:20,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:20,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:20,345 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:20,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:22,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:23,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:24,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:24,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:24,976 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:24,981 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:25,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:26,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:28,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:28,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:28,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:28,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:31,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:31,442 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:31,442 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:31,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:32,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:32,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:34,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:35,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:35,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:35,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:35,437 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:36,420 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:36,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:38,760 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:34:39,761 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:39,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:39,993 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:39,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:39,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:40,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:42,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:43,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:43,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:43,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:45,563 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:46,539 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:46,539 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:46,540 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:46,545 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:47,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:48,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:50,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:51,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:51,100 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:51,100 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:51,105 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:51,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:52,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:56,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:58,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:58,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:58,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:00,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:00,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:00,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:00,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:00,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:01,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:02,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:04,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:04,874 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:04,875 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:04,880 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:05,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:06,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:06,880 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:08,762 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:35:09,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:09,696 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:09,696 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:09,701 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:10,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:12,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:12,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:13,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:13,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:13,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:14,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:14,546 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:14,546 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:14,551 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:15,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:16,458 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:18,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:19,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:19,535 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:19,535 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:19,540 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:20,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:20,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:24,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:27,011 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:27,012 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:27,012 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:27,017 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:27,467 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:28,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:28,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:28,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:28,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:29,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:33,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:33,131 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:33,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:33,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:33,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:34,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:35,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:38,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:35:39,847 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:39,848 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:39,848 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:39,853 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:40,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:40,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:42,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:43,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:43,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:43,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:44,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:44,726 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:44,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:44,731 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:45,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:46,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:46,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:49,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:49,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:49,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:49,183 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:49,488 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:50,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:52,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:53,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:53,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:53,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:53,641 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:54,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:54,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:57,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:58,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:58,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:58,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:03,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:03,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:03,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:03,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:03,348 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:03,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:04,501 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:07,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:07,706 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:07,706 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:07,710 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:08,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:08,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:08,766 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:36:10,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:12,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:12,445 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:12,445 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:12,450 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:12,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:13,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:13,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:13,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:14,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:14,596 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:16,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:16,967 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:16,967 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:16,972 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:17,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:18,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:20,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:23,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:23,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:23,030 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:23,035 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:23,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:24,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:25,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:27,291 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:27,292 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:27,292 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:27,297 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:27,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:28,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:28,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:28,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:28,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:30,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:32,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:32,180 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:32,180 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:32,185 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:32,530 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:34,533 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:36,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:36,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:36,853 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:36,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:36,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:37,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:38,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:38,772 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:36:41,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:43,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:43,093 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:43,093 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:43,098 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:43,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:43,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:43,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:43,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:44,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:47,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:49,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:49,146 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:49,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:49,151 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:49,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:50,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:53,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:54,232 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:54,233 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:54,233 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:54,238 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:54,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:56,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:58,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:58,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:58,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:58,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:59,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:59,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:59,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:59,919 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:00,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:02,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:03,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:04,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:04,467 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:04,467 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:04,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:04,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:06,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:08,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:37:08,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:08,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:08,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:08,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:09,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:09,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:10,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:13,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:13,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:13,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:14,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:14,147 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:14,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:14,152 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:14,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:15,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:16,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:18,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:18,583 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:18,583 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:18,589 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:19,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:20,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:20,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:24,765 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:24,766 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:24,766 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:24,771 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:25,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:25,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:26,589 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:28,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:28,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:28,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:30,483 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:30,484 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:30,484 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:30,489 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:30,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:31,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:32,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:35,746 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:35,747 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:35,747 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:35,752 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:36,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:36,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:36,752 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:38,773 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:37:40,180 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:40,181 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:40,181 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:40,186 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:40,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:42,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:42,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:43,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:43,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:43,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:44,699 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:44,699 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:44,700 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:44,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:45,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:46,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:47,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:49,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:49,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:49,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:49,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:49,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:50,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:53,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:53,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:53,454 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:53,454 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:53,459 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:53,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:54,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:58,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:58,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:58,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:58,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:00,568 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:00,568 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:00,568 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:00,574 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:00,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:02,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:03,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:06,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:06,043 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:06,043 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:06,048 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:06,628 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:08,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:08,775 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:38:08,776 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:13,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:13,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:13,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:14,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:14,413 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:14,413 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:14,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:14,418 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:14,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:16,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:19,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:19,004 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:19,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:19,009 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:19,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:20,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:20,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:23,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:23,510 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:23,510 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:23,515 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:23,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:24,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:25,516 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:27,699 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:27,700 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:27,700 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:27,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:28,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:28,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:28,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:28,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:28,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:30,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:33,920 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:33,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:33,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:33,926 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:34,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:35,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:36,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:38,777 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:38:39,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:39,387 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:39,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:39,392 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:39,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:40,672 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:41,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:43,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:43,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:43,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:43,690 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:43,690 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:43,690 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:43,698 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:44,675 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:44,675 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:46,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:50,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:50,319 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:50,319 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:50,324 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:50,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:52,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:52,682 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:55,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:55,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:55,246 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:55,251 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:55,684 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:56,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:58,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:58,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:58,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:58,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:59,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:59,756 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:59,756 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:59,761 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:00,692 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:00,692 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:03,797 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:06,723 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:06,724 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:06,724 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:06,729 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:07,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:08,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:08,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:39:09,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:11,296 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:11,297 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:11,297 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:11,302 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:11,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:12,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:13,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:13,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:13,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:15,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:15,407 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:15,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:15,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:15,412 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:15,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:16,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:20,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:21,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:21,070 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:21,070 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:21,075 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:21,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:22,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:25,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:25,998 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:25,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:25,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:26,003 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:26,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:28,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:28,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:28,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:28,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:31,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:32,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:32,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:32,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:32,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:32,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:34,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:36,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:37,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:37,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:37,256 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:37,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:37,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:38,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:38,782 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:39:41,748 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:41,749 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:41,749 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:41,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:41,754 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:42,733 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:42,733 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:43,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:43,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:43,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:46,563 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:46,563 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:46,564 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:46,568 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:46,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:47,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:48,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:52,606 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:54,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:54,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:54,950 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:54,955 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:55,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:56,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:57,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:58,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:58,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:58,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:59,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:59,471 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:59,471 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:59,476 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:59,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:00,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:03,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:03,532 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:03,533 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:03,533 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:03,539 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:03,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:04,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:07,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:07,698 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:07,699 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:07,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:07,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:08,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:08,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:08,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:40:13,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:13,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:13,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:13,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:13,617 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:13,617 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:13,623 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:13,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:14,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:14,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:19,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:22,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:22,008 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:22,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:22,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:22,769 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:24,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:25,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:26,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:26,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:26,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:26,949 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:27,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:28,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:28,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:28,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:28,775 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:30,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:31,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:31,339 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:31,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:31,344 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:31,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:32,780 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:35,724 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:35,725 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:35,726 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:35,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:35,731 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:35,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:36,784 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:38,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:40:40,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:40,018 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:40,019 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:40,024 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:40,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:41,024 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:42,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:43,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:43,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:43,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:44,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:44,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:44,674 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:44,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:44,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:46,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:46,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:49,185 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:49,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:49,186 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:49,191 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:49,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:50,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:52,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:53,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:53,579 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:53,579 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:53,584 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:53,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:54,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:57,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:58,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:58,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:58,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:58,956 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:58,956 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:58,957 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:58,962 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:59,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:00,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:02,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:07,443 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:07,444 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:07,444 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:07,449 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:07,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:08,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:08,789 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:41:08,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:12,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:12,602 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:12,602 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:12,607 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:12,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:13,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:13,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:13,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:13,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:14,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:17,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:17,035 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:17,035 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:17,040 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:17,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:18,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:19,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:21,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:21,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:21,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:21,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:22,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:22,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:24,860 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:28,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:28,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:28,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:30,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:30,386 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:30,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:30,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:30,392 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:30,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:32,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:35,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:36,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:36,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:36,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:36,170 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:36,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:38,791 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:41:38,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:40,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:42,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:42,361 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:42,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:42,366 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:42,847 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:43,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:43,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:43,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:44,850 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:46,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:46,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:46,847 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:46,847 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:46,852 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:46,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:48,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:50,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:50,975 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:50,975 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:50,980 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:51,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:51,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:52,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:55,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:55,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:55,675 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:55,680 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:55,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:56,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:57,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:58,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:58,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:58,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:00,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:00,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:00,227 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:00,232 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:00,867 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:02,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:03,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:06,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:06,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:06,299 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:06,304 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:06,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:08,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:08,793 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:42:08,872 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:12,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:12,589 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:12,589 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:12,594 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:12,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:13,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:13,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:13,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:13,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:14,879 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:18,595 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:20,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:20,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:20,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:20,949 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:21,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:22,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:23,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:25,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:25,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:25,196 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:25,201 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:25,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:26,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:28,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:28,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:28,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:29,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:31,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:31,609 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:31,609 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:31,614 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:31,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:32,896 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:34,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:36,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:36,059 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:36,059 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:36,064 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:36,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:36,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:38,798 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:42:39,799 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:43,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:43,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:43,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:44,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:44,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:44,534 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:44,539 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:44,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:45,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:46,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:50,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:51,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:51,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:51,315 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:51,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:51,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:52,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:55,178 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:55,179 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:55,179 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:55,184 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:55,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:56,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:56,917 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:58,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:58,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:58,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:01,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:01,144 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:01,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:01,149 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:01,185 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:01,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:02,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:06,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:08,797 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:43:09,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:09,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:09,674 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:09,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:09,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:10,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:11,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:13,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:13,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:13,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:14,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:14,036 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:14,036 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:14,041 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:14,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:14,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:17,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:19,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:19,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:19,299 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:19,303 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:19,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:20,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:22,304 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:23,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:23,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:23,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:23,345 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:23,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:24,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:27,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:28,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:28,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:28,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:28,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:28,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:28,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:28,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:28,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:30,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:32,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:34,748 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:34,748 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:34,749 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:34,754 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:34,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:36,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:37,754 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:38,802 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:43:38,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:38,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:38,926 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:38,931 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:38,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:40,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:42,931 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:43,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:43,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:43,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:43,672 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:43,672 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:43,673 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:43,677 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:43,961 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:46,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:48,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:52,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:52,207 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:52,207 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:52,211 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:52,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:54,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:54,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:58,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:58,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:58,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:59,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:00,596 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:00,597 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:00,597 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:00,601 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:00,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:02,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:04,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:04,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:04,705 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:04,705 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:04,709 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:04,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:06,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:08,802 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:44:09,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:09,203 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:09,203 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:09,208 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:09,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:10,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:10,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:13,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:13,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:13,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:14,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:14,347 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:14,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:14,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:15,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:15,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:17,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:18,760 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:18,761 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:18,761 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:18,766 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:19,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:20,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:21,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:24,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:24,457 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:24,457 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:24,462 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:25,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:26,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:27,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:28,320 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:28,320 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:28,320 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:28,326 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:28,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:28,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:28,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:29,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:31,019 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:31,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:32,538 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:32,538 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:32,539 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:32,543 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:33,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:35,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:37,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:38,804 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:44:41,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:41,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:41,044 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:41,049 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:42,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:43,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:43,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:43,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:43,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:43,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:46,095 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:46,096 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:46,096 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:46,101 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:47,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:48,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:49,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:53,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:53,185 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:53,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:53,186 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:53,190 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:54,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:55,040 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:57,950 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:57,951 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:57,951 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:57,955 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:58,042 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:58,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:58,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:58,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:58,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:01,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:03,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:03,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:03,030 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:03,035 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:03,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:04,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:05,051 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:08,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:08,741 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:08,741 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:08,746 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:08,806 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:45:09,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:09,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:11,057 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:13,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:13,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:13,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:15,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:15,250 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:15,251 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:15,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:15,256 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:16,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:17,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:20,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:21,220 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:21,220 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:21,221 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:21,225 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:22,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:23,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:26,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:28,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:28,032 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:28,033 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:28,037 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:28,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:28,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:28,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:28,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:31,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:31,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:34,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:34,645 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:34,645 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:34,651 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:35,081 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:36,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:37,084 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:38,809 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:45:40,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:40,121 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:40,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:40,127 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:41,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:42,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:43,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:43,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:43,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:43,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:45,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:45,236 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:45,237 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:45,241 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:46,092 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:47,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:47,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:49,524 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:49,525 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:49,525 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:49,530 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:50,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:51,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:52,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:54,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:54,958 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:54,958 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:54,963 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:55,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:57,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:57,964 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:58,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:58,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:58,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:00,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:00,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:00,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:00,641 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:01,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:03,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:03,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:06,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:06,685 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:06,685 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:06,690 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:07,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:08,690 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:08,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:46:09,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:11,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:11,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:11,212 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:11,217 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:12,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:13,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:13,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:13,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:13,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:14,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:16,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:16,396 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:16,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:16,401 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:17,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:19,128 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:20,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:21,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:21,855 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:21,855 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:21,860 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:22,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:25,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:25,860 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:28,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:28,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:28,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:28,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:28,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:28,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:28,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:29,137 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:31,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:31,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:34,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:34,313 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:34,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:34,318 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:35,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:37,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:37,319 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:38,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:38,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:38,813 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:38,818 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:38,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:46:39,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:41,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:42,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:43,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:43,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:43,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:45,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:45,469 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:45,470 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:45,474 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:46,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:47,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:48,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:49,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:49,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:49,711 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:49,715 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:50,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:51,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:53,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:53,867 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:53,868 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:53,868 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:53,873 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:54,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:55,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:58,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:58,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:58,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:59,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:59,371 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:59,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:59,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:59,376 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:00,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:01,167 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:04,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:04,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:04,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:04,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:04,919 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:05,169 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:07,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:08,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:47:09,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:09,329 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:09,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:09,333 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:10,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:10,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:11,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:13,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:13,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:13,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:15,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:17,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:17,315 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:17,316 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:17,320 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:18,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:19,187 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:21,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:22,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:22,234 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:22,234 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:22,239 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:23,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:25,195 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:26,622 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:26,622 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:26,623 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:26,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:26,628 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:27,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:28,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:28,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:28,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:29,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:30,940 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:30,941 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:30,941 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:30,946 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:31,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:31,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:33,209 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:35,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:35,206 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:35,206 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:35,212 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:35,213 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:37,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:37,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:38,817 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:47:39,725 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:39,726 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:39,726 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:39,731 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:40,217 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:41,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:42,731 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:43,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:43,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:43,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:45,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:45,701 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:45,701 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:45,706 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:46,226 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:47,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:47,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:50,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:50,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:50,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:50,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:50,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:53,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:53,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:56,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:56,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:56,790 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:56,795 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:57,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:58,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:58,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:58,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:58,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:59,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:02,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:02,687 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:02,688 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:02,692 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:03,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:03,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:05,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:08,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:08,819 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:48:09,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:09,621 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:09,621 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:09,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:10,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:11,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:13,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:13,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:13,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:14,262 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:14,263 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:14,263 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:14,263 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:14,268 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:14,269 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:17,274 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:19,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:19,181 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:19,181 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:19,187 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:19,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:20,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:21,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:23,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:23,606 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:23,606 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:23,611 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:24,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:25,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:25,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:28,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:28,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:29,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:29,847 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:29,847 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:29,852 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:30,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:30,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:31,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:34,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:34,732 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:34,732 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:34,737 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:35,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:36,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:37,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:38,821 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:48:39,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:39,106 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:39,107 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:39,111 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:39,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:41,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:42,112 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:43,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:43,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:43,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:44,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:44,021 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:44,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:44,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:44,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:47,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:48,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:51,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:51,270 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:51,270 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:51,275 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:51,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:53,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:53,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:58,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:58,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:58,283 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:58,284 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:58,288 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:58,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:58,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:58,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:58,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:01,341 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:02,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:02,749 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:02,749 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:02,754 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:03,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:03,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:05,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:06,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:06,581 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:06,582 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:06,587 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:07,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:08,824 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:49:08,825 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:09,353 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:13,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:13,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:13,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:14,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:15,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:15,429 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:15,429 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:15,435 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:16,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:17,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:20,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:24,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:24,132 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:24,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:24,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:24,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:26,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:27,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:28,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:28,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:29,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:29,630 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:29,631 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:29,635 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:30,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:31,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:31,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:35,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:35,261 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:35,261 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:35,266 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:35,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:37,266 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:37,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:38,829 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:49:41,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:41,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:41,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:41,519 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:42,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:42,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:43,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:43,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:43,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:43,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:46,219 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:46,220 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:46,220 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:46,225 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:46,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:48,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:49,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:51,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:51,108 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:51,108 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:51,113 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:51,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:53,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:54,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:55,868 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:55,869 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:55,869 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:55,874 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:56,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:57,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:58,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:58,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:58,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:59,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:00,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:00,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:00,315 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:00,320 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:00,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:03,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:05,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:06,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:06,008 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:06,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:06,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:06,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:07,438 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:08,828 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:50:10,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:10,510 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:10,511 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:10,511 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:10,516 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:11,442 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:13,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:13,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:13,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:13,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:15,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:15,880 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:15,880 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:15,881 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:15,886 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:16,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:17,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:20,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:20,197 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:20,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:20,202 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:20,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:21,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:23,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:26,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:26,815 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:26,816 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:26,816 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:26,821 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:27,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:28,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:28,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:29,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:31,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:35,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:35,159 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:35,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:35,165 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:35,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:37,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:37,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:38,833 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:50:39,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:39,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:39,034 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:39,038 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:39,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:41,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:43,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:43,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:43,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:43,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:44,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:44,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:44,312 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:44,316 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:44,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:47,496 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:48,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:50,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:50,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:50,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:50,247 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:50,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:53,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:54,247 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:58,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:58,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:58,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:58,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:58,613 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:58,614 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:58,618 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:59,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:59,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:01,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:04,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:08,832 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:51:09,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:10,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:10,035 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:10,035 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:10,039 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:10,527 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:51:11,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:13,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:13,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:13,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:15,087 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:15,088 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:15,088 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:15,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:15,092 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:15,536 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:51:17,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:20,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:20,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:20,586 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:20,586 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:20,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:21,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:51:23,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:25,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:25,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:25,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:25,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:25,283 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:25,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:51:27,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:28,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:28,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:28,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:30,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:33,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:33,677 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:33,678 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:33,682 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:34,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:51:35,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:35,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:38,835 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:51:40,836 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:41,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:41,301 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:41,301 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:41,306 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:41,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:51:43,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:43,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:43,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:43,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:46,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:48,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:48,342 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:48,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:48,347 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:48,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:51:51,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:52,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:53,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:53,433 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:53,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:53,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:53,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:51:55,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:57,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:58,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:58,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:58,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:58,924 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:58,925 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:58,925 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:58,930 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:59,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:01,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:02,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:03,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:03,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:03,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:03,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:04,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:05,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:08,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:08,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:08,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:08,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:08,214 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:08,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:08,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:52:09,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:13,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:13,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:13,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:13,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:13,573 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:13,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:13,573 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:13,578 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:13,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:15,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:18,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:19,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:19,258 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:19,258 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:19,262 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:19,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:21,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:24,263 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:28,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:28,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:28,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:28,643 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:28,648 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:28,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:29,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:31,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:33,103 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:33,104 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:33,105 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:33,109 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:33,655 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:35,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:35,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:38,839 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:52:39,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:39,603 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:39,603 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:39,608 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:39,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:40,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:41,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:43,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:43,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:43,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:45,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:45,035 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:45,035 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:45,040 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:45,675 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:46,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:47,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:50,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:50,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:50,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:50,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:50,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:51,319 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:53,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:54,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:54,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:54,969 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:54,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:55,690 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:56,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:57,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:58,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:58,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:58,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:00,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:00,723 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:00,723 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:00,728 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:01,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:02,765 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:03,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:06,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:06,138 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:06,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:06,143 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:06,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:07,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:08,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:08,844 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:53:11,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:11,201 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:11,201 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:11,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:11,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:13,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:13,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:13,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:13,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:13,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:16,074 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:16,075 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:16,075 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:16,080 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:16,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:17,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:19,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:20,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:20,369 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:20,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:20,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:20,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:23,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:24,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:25,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:25,243 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:25,243 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:25,249 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:25,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:27,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:28,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:29,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:29,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:29,762 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:29,763 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:29,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:30,741 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:31,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:34,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:34,437 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:34,437 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:34,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:34,747 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:35,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:37,754 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:38,844 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:53:40,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:40,364 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:40,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:40,369 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:40,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:41,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:43,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:43,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:43,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:43,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:45,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:45,236 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:45,236 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:45,241 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:45,763 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:47,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:47,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:50,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:50,834 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:50,834 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:50,839 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:51,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:52,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:53,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:55,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:55,635 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:55,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:55,641 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:55,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:57,784 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:58,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:58,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:58,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:58,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:00,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:00,409 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:00,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:00,415 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:00,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:03,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:04,454 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:06,936 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:06,937 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:06,937 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:06,942 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:07,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:08,849 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:54:09,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:09,850 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:11,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:11,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:11,448 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:11,453 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:11,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:13,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:13,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:13,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:13,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:15,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:16,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:16,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:16,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:16,012 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:16,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:16,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:21,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:21,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:21,371 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:21,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:21,376 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:21,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:22,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:25,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:25,626 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:25,626 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:25,631 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:25,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:26,632 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:26,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:28,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:28,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:30,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:30,372 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:30,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:30,377 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:30,831 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:32,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:32,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:34,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:34,594 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:34,594 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:34,599 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:34,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:36,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:37,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:38,848 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:54:38,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:38,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:38,879 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:38,884 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:39,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:40,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:42,884 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:43,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:43,106 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:43,106 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:43,111 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:43,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:43,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:43,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:43,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:44,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:48,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:48,918 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:48,919 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:48,919 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:48,924 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:49,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:50,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:52,971 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:52,972 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:52,972 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:52,977 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:53,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:53,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:54,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:58,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:58,004 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:58,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:58,009 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:58,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:58,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:58,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:58,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:58,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:59,744 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:04,744 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:05,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:05,270 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:05,271 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:05,276 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:05,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:06,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:08,850 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:55:09,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:09,493 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:09,493 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:09,498 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:09,889 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:10,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:10,892 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:13,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:13,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:13,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:15,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:17,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:17,819 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:17,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:17,824 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:17,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:18,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:20,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:21,867 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:21,868 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:21,869 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:21,874 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:21,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:22,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:25,875 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:26,804 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:26,805 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:26,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:26,811 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:26,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:28,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:28,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:28,919 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:31,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:31,339 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:31,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:31,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:31,345 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:31,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:32,925 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:36,108 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:36,112 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:36,113 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:36,118 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:36,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:36,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:37,118 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:38,852 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:55:40,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:40,754 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:40,754 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:40,759 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:40,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:42,760 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:42,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:43,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:43,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:43,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:45,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:45,000 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:45,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:45,005 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:45,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:46,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:48,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:49,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:49,243 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:49,243 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:49,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:49,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:50,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:53,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:55,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:55,102 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:55,102 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:55,107 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:55,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:56,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:58,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:58,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:58,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:58,563 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:59,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:59,662 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:59,662 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:59,667 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:59,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:00,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:03,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:04,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:04,602 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:04,602 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:04,607 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:04,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:06,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:08,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:56:08,855 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:09,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:09,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:09,013 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:09,018 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:09,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:10,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:13,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:13,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:13,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:13,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:13,685 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:13,686 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:13,690 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:13,985 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:14,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:14,988 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:18,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:18,551 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:18,551 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:18,556 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:18,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:20,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:20,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:23,720 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:23,721 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:23,721 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:23,726 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:23,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:25,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:25,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:28,296 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:28,296 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:28,296 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:28,301 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:28,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:28,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:28,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:29,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:29,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:31,632 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:32,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:32,645 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:32,645 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:32,650 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:33,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:35,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:36,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:38,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:38,171 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:38,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:38,176 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:38,860 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:56:39,022 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:39,022 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:41,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:43,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:43,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:43,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:44,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:44,353 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:44,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:44,358 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:45,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:45,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:47,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:49,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:49,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:49,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:49,081 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:50,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:51,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:53,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:54,262 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:54,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:54,263 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:54,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:55,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:55,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:58,268 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:58,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:58,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:58,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:58,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:58,558 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:58,558 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:58,563 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:59,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:01,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:03,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:06,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:06,592 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:06,593 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:06,597 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:07,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:08,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:08,858 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:57:09,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:10,991 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:10,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:10,992 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:10,997 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:11,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:13,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:13,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:13,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:13,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:14,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:15,714 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:15,715 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:15,715 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:15,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:16,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:17,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:19,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:20,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:20,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:20,650 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:20,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:21,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:23,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:25,099 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:25,100 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:25,100 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:25,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:25,104 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:26,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:27,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:28,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:28,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:28,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:29,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:29,397 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:29,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:29,402 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:30,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:30,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:31,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:34,108 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:34,108 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:34,109 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:34,113 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:35,105 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:35,105 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:36,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:38,812 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:38,812 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:38,813 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:38,817 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:38,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:57:39,109 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:41,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:41,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:43,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:43,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:43,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:43,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:43,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:43,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:43,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:44,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:45,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:47,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:48,496 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:48,496 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:48,497 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:48,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:49,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:49,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:53,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:56,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:56,313 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:56,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:56,318 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:57,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:57,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:58,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:58,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:58,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:58,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:02,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:02,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:02,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:02,174 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:03,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:03,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:04,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:08,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:08,586 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:08,586 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:08,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:08,863 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:58:09,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:09,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:09,864 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:13,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:13,337 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:13,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:13,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:13,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:13,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:13,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:14,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:15,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:15,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:18,850 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:18,850 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:18,851 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:18,855 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:19,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:20,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:21,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:24,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:24,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:24,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:24,732 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:25,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:26,732 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:27,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:28,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:28,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:28,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:29,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:29,277 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:29,277 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:29,282 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:30,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:31,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:32,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:36,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:36,423 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:36,423 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:36,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:37,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:37,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:37,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:38,865 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:58:41,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:41,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:41,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:41,868 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:42,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:42,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:43,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:43,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:43,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:43,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:47,812 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:47,816 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:47,816 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:47,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:48,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:48,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:49,217 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:52,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:52,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:52,465 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:52,470 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:53,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:53,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:54,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:58,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:58,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:58,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:59,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:59,179 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:59,179 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:59,185 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:59,231 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:00,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:01,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:03,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:03,587 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:03,587 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:03,592 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:04,238 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:05,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:05,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:08,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:08,134 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:08,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:08,139 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:08,245 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:08,868 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:59:09,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:10,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:12,812 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:12,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:12,813 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:12,818 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:13,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:13,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:13,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:13,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:15,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:16,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:18,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:18,518 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:18,518 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:18,524 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:19,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:19,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:22,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:23,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:23,762 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:23,762 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:23,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:24,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:25,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:27,768 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:28,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:28,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:28,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:28,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:28,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:28,619 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:28,624 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:29,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:29,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:33,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:33,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:33,670 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:33,670 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:33,675 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:34,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:35,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:38,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:38,243 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:38,243 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:38,249 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:38,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:38,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:59:38,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:39,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:43,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:43,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:43,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:43,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:43,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:43,969 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:43,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:43,982 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:44,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:45,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:48,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:48,266 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:48,266 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:48,271 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:48,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:49,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:49,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:53,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:53,081 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:53,082 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:53,086 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:53,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:55,087 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:55,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:57,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:57,308 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:57,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:57,313 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:57,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:58,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:58,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:58,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:59,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:00,546 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:02,503 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:02,503 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:02,504 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:02,508 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:03,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:03,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:06,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:06,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:06,903 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:06,904 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:06,908 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:07,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:08,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:00:09,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:11,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:11,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:11,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:11,059 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:11,341 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:12,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:13,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:13,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:13,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:13,346 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:15,864 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:15,865 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:15,865 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:15,870 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:16,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:17,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:17,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:22,906 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:23,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:23,433 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:23,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:23,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:24,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:25,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:28,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:28,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:28,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:28,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:28,624 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:28,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:28,624 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:28,629 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:29,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:29,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:33,524 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:33,525 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:33,525 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:33,530 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:34,376 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:34,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:35,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:37,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:37,997 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:37,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:38,002 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:38,382 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:38,874 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:00:39,385 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:39,875 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:42,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:42,237 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:42,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:42,242 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:42,388 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:43,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:43,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:43,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:43,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:45,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:47,264 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:47,265 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:47,265 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:47,270 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:47,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:49,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:51,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:51,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:51,379 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:51,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:51,384 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:51,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:53,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:56,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:56,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:56,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:56,476 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:56,480 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:57,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:57,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:58,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:58,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:58,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:01,538 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:01,714 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:01,714 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:01,715 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:01,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:02,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:03,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:06,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:07,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:07,454 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:07,454 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:07,459 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:08,427 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:08,879 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:01:09,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:11,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:12,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:12,325 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:12,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:12,331 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:12,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:13,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:13,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:13,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:13,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:17,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:18,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:18,533 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:18,533 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:18,538 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:19,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:19,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:22,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:22,941 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:22,942 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:22,942 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:22,946 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:23,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:25,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:27,947 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:28,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:28,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:28,243 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:28,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:28,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:28,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:28,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:28,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:29,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:33,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:36,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:36,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:36,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:36,814 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:37,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:37,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:38,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:38,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:01:43,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:43,171 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:43,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:43,176 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:43,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:43,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:43,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:43,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:44,723 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:45,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:48,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:48,309 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:48,309 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:48,314 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:48,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:49,488 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:50,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:53,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:53,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:53,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:53,028 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:53,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:55,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:56,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:58,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:58,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:58,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:58,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:58,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:58,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:58,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:59,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:59,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:01,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:04,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:04,162 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:04,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:04,167 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:04,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:05,505 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:07,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:08,881 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:02:10,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:10,383 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:10,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:10,388 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:10,508 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:11,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:12,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:13,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:13,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:13,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:17,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:17,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:17,066 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:17,071 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:17,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:18,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:19,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:23,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:23,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:23,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:23,028 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:23,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:24,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:25,527 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:27,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:27,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:27,664 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:27,670 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:28,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:28,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:28,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:28,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:29,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:29,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:34,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:34,295 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:34,295 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:34,300 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:34,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:35,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:35,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:38,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:02:39,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:39,040 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:39,040 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:39,045 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:39,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:41,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:41,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:43,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:43,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:43,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:43,544 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:43,580 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:43,580 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:43,585 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:44,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:45,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:46,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:50,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:50,565 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:50,565 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:50,570 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:51,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:51,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:51,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:56,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:56,453 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:56,453 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:56,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:56,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:57,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:57,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:58,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:58,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:58,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:02,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:04,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:04,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:04,954 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:04,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:05,567 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:07,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:07,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:08,885 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:03:13,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:13,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:13,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:13,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:13,600 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:13,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:13,601 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:13,606 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:14,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:15,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:18,606 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:19,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:19,542 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:19,542 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:19,547 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:19,581 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:21,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:24,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:27,048 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:27,049 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:27,049 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:27,054 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:27,588 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:28,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:28,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:28,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:29,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:29,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:30,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:30,887 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:30,887 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:30,892 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:31,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:31,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:34,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:35,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:35,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:35,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:35,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:35,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:37,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:38,891 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:03:40,204 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:40,205 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:40,205 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:40,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:40,210 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:40,603 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:41,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:43,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:43,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:43,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:45,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:46,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:46,259 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:46,259 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:46,265 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:46,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:47,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:51,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:51,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:51,628 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:51,628 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:51,634 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:52,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:53,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:56,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:56,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:56,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:56,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:56,845 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:57,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:57,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:58,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:58,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:58,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:01,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:01,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:01,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:01,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:01,746 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:02,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:03,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:06,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:08,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:08,095 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:08,095 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:08,100 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:08,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:08,890 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:04:09,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:11,891 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:13,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:13,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:13,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:13,169 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:13,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:13,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:13,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:13,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:15,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:17,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:17,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:17,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:17,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:17,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:17,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:19,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:22,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:22,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:22,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:22,162 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:22,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:23,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:23,652 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:27,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:27,259 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:27,259 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:27,264 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:27,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:28,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:28,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:28,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:28,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:29,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:32,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:32,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:32,047 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:32,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:32,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:33,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:34,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:36,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:36,503 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:36,503 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:36,508 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:36,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:37,666 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:38,895 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:04:39,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:43,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:43,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:43,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:44,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:44,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:44,820 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:44,825 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:45,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:45,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:45,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:49,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:49,348 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:49,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:49,353 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:49,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:51,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:51,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:54,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:54,511 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:54,511 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:54,517 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:54,682 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:55,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:56,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:58,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:58,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:58,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:58,471 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:58,653 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:58,653 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:58,658 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:58,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:59,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:01,659 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:04,835 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:04,836 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:04,837 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:04,842 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:05,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:05,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:06,842 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:08,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:05:09,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:09,289 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:09,289 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:09,295 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:09,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:11,701 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:12,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:13,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:13,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:13,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:13,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:13,561 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:13,561 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:13,566 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:13,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:15,707 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:17,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:19,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:19,555 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:19,556 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:19,561 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:19,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:21,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:23,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:24,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:24,738 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:24,738 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:24,743 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:25,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:25,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:28,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:28,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:28,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:28,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:28,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:28,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:28,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:28,813 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:29,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:29,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:33,703 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:33,704 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:33,705 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:33,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:33,710 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:33,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:35,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:38,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:38,199 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:38,199 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:38,203 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:38,733 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:38,900 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:05:38,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:39,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:42,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:42,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:42,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:42,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:43,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:43,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:43,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:43,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:43,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:44,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:47,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:47,389 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:47,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:47,395 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:47,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:49,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:50,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:54,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:54,476 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:54,476 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:54,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:54,747 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:55,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:55,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:58,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:58,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:58,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:58,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:58,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:58,870 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:58,875 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:59,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:59,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:00,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:05,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:05,092 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:05,093 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:05,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:05,760 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:05,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:06,098 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:08,899 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:06:09,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:09,785 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:09,785 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:09,791 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:10,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:11,766 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:11,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:13,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:13,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:13,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:14,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:14,163 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:14,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:14,168 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:14,769 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:15,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:17,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:18,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:18,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:18,702 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:18,708 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:18,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:19,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:22,708 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:25,055 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:25,055 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:25,056 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:25,061 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:25,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:25,784 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:28,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:28,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:28,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:28,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:30,054 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:30,055 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:30,055 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:30,061 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:30,787 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:31,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:33,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:34,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:34,555 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:34,555 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:34,560 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:34,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:35,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:38,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:38,900 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:06:40,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:40,934 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:40,934 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:40,939 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:41,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:41,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:43,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:43,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:43,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:44,544 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:47,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:47,137 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:47,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:47,143 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:47,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:47,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:50,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:51,468 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:51,469 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:51,469 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:51,474 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:51,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:53,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:55,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:56,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:56,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:56,767 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:56,772 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:56,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:57,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:58,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:58,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:58,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:00,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:01,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:01,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:01,883 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:01,888 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:02,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:03,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:05,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:07,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:07,805 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:07,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:07,810 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:07,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:08,903 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:07:09,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:11,904 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:13,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:13,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:13,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:14,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:14,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:14,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:14,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:15,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:15,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:16,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:19,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:19,367 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:19,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:19,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:19,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:21,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:22,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:24,836 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:24,837 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:24,837 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:24,842 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:24,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:25,847 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:27,842 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:28,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:28,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:28,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:29,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:29,126 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:29,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:29,132 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:29,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:29,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:33,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:33,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:33,793 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:33,794 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:33,799 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:33,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:35,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:38,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:38,905 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:07:40,266 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:40,266 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:40,266 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:40,272 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:40,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:41,864 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:43,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:43,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:43,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:44,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:49,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:49,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:49,877 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:49,877 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:49,883 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:50,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:51,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:54,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:55,048 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:55,048 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:55,048 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:55,053 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:55,878 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:55,878 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:58,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:58,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:58,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:59,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:59,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:59,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:59,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:59,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:59,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:00,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:02,848 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:02,848 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:02,848 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:02,854 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:02,884 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:03,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:05,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:07,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:07,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:07,964 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:07,969 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:08,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:08,911 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:08:09,892 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:10,912 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:12,812 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:12,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:12,813 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:12,818 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:12,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:13,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:13,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:13,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:13,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:16,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:16,927 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:16,928 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:16,928 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:16,933 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:17,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:17,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:21,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:22,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:22,450 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:22,451 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:22,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:22,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:23,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:27,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:28,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:28,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:28,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:29,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:29,132 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:29,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:29,137 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:29,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:29,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:33,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:34,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:34,610 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:34,610 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:34,615 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:34,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:35,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:38,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:38,909 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:08:39,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:39,152 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:39,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:39,157 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:39,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:39,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:43,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:43,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:43,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:43,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:43,629 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:43,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:43,629 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:43,635 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:43,928 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:45,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:48,103 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:48,104 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:48,104 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:48,109 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:48,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:49,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:49,936 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:54,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:55,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:55,147 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:55,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:55,152 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:55,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:55,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:58,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:58,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:58,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:59,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:00,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:00,938 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:00,938 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:00,943 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:00,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:01,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:04,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:05,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:05,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:05,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:05,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:05,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:07,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:08,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:09:10,916 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:11,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:11,191 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:11,191 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:11,196 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:11,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:11,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:13,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:13,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:13,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:16,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:16,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:16,679 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:16,679 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:16,684 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:16,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:17,967 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:21,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:22,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:22,356 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:22,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:22,361 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:22,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:23,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:26,843 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:26,843 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:26,844 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:26,844 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:26,849 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:26,974 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:27,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:28,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:28,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:28,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:30,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:30,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:30,779 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:30,783 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:30,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:31,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:32,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:35,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:35,578 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:35,578 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:35,583 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:35,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:37,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:38,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:38,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:09:41,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:41,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:41,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:41,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:41,988 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:43,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:43,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:43,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:43,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:43,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:47,047 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:47,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:47,048 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:47,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:47,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:47,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:49,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:54,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:55,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:55,460 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:55,460 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:55,464 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:56,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:58,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:58,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:58,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:58,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:59,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:03,488 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:03,489 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:03,489 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:03,494 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:04,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:05,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:06,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:07,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:07,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:07,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:07,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:08,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:08,916 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:10:10,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:10,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:13,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:13,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:13,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:14,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:14,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:14,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:14,776 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:15,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:16,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:16,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:20,688 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:20,689 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:20,689 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:20,694 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:21,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:22,029 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:22,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:26,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:26,488 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:26,488 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:26,493 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:27,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:28,035 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:28,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:28,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:28,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:28,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:30,884 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:30,885 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:30,885 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:30,890 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:31,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:32,040 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:33,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:36,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:36,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:36,791 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:36,795 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:37,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:38,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:38,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:10:39,919 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:43,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:43,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:43,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:45,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:45,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:45,223 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:45,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:45,228 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:46,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:46,053 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:49,735 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:49,736 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:49,736 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:49,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:50,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:50,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:52,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:54,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:54,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:54,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:54,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:55,061 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:56,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:56,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:58,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:58,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:58,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:00,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:00,634 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:00,634 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:00,639 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:01,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:01,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:02,069 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:06,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:08,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:08,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:08,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:08,733 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:08,920 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:11:09,073 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:10,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:11,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:13,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:13,180 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:13,181 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:13,185 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:13,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:13,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:13,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:14,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:14,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:17,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:19,185 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:19,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:19,186 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:19,191 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:20,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:20,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:23,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:23,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:23,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:23,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:23,440 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:24,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:24,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:28,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:28,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:28,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:28,100 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:28,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:28,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:28,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:28,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:28,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:30,105 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:33,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:33,535 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:33,535 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:33,541 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:34,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:34,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:36,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:37,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:37,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:37,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:37,892 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:38,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:38,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:11:39,927 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:40,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:42,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:42,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:42,969 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:42,975 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:43,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:43,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:43,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:43,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:44,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:45,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:47,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:47,593 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:47,594 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:47,599 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:48,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:50,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:50,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:52,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:52,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:52,436 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:52,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:53,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:54,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:56,443 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:57,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:57,365 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:57,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:57,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:58,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:58,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:58,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:58,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:58,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:01,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:01,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:01,990 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:01,990 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:01,996 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:02,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:04,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:06,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:06,323 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:06,323 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:06,329 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:07,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:07,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:08,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:08,925 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:12:11,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:11,601 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:11,601 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:11,606 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:12,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:12,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:13,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:13,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:13,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:14,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:16,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:16,726 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:16,726 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:16,732 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:17,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:17,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:18,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:22,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:25,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:25,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:25,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:25,156 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:25,169 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:26,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:28,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:28,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:28,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:28,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:29,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:29,463 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:29,463 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:29,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:30,178 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:30,178 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:33,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:35,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:35,294 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:35,294 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:35,300 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:36,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:36,185 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:38,931 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:12:38,932 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:39,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:39,910 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:39,910 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:39,916 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:40,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:42,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:43,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:43,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:43,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:44,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:44,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:44,630 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:44,630 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:44,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:45,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:46,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:49,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:53,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:53,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:53,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:53,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:53,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:54,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:55,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:58,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:58,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:58,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:00,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:01,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:01,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:01,643 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:01,650 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:02,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:04,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:05,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:06,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:06,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:06,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:06,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:07,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:08,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:08,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:13:10,930 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:13,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:13,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:13,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:14,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:14,505 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:14,506 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:14,512 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:15,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:16,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:16,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:20,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:20,435 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:20,435 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:20,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:21,238 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:22,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:22,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:25,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:25,125 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:25,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:25,131 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:25,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:26,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:28,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:28,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:28,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:28,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:29,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:29,002 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:29,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:29,009 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:29,250 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:30,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:33,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:33,313 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:33,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:33,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:33,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:34,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:34,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:38,319 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:38,931 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:13:39,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:39,589 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:39,589 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:39,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:40,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:40,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:43,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:43,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:43,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:43,575 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:44,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:44,464 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:44,464 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:44,470 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:45,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:46,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:49,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:49,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:49,824 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:49,824 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:49,831 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:50,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:52,283 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:54,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:56,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:56,145 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:56,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:56,151 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:56,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:58,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:58,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:58,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:58,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:00,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:00,413 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:00,413 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:00,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:00,419 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:01,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:02,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:05,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:05,816 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:05,817 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:05,817 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:05,823 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:06,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:08,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:08,934 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:14:09,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:09,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:09,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:09,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:10,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:10,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:12,307 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:13,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:13,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:13,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:14,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:14,882 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:14,882 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:14,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:15,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:15,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:16,323 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:19,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:19,653 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:19,653 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:19,660 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:20,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:20,338 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:21,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:24,064 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:24,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:24,065 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:24,074 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:24,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:26,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:27,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:28,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:28,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:28,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:30,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:30,556 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:30,556 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:30,563 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:31,359 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:32,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:32,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:35,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:35,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:35,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:35,444 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:36,373 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:36,373 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:38,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:38,936 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:14:40,755 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:40,755 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:40,755 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:40,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:41,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:42,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:43,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:43,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:43,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:43,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:44,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:44,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:44,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:44,893 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:45,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:46,391 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:48,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:49,099 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:49,100 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:49,100 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:49,108 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:49,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:50,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:53,779 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:53,780 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:53,780 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:53,794 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:54,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:54,795 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:56,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:58,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:58,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:58,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:58,855 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:58,856 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:58,856 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:58,868 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:59,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:59,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:00,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:03,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:03,570 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:03,570 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:03,581 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:04,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:04,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:05,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:07,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:07,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:07,949 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:07,957 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:08,435 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:08,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:15:10,442 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:10,950 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:13,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:13,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:13,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:13,728 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:13,729 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:13,729 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:13,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:14,445 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:14,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:16,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:18,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:18,182 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:18,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:18,195 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:18,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:20,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:22,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:24,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:24,470 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:24,471 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:24,480 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:25,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:26,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:27,480 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:28,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:28,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:28,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:29,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:29,971 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:29,971 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:29,978 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:30,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:32,521 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:33,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:38,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:38,940 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:15:39,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:39,392 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:39,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:39,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:39,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:40,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:43,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:43,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:43,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:43,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:44,052 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:44,053 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:44,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:44,061 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:44,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:46,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:49,098 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:49,419 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:49,420 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:49,420 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:49,429 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:49,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:50,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:54,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:55,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:55,123 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:55,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:55,131 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:55,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:56,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:58,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:58,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:58,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:59,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:00,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:00,133 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:00,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:00,151 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:00,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:02,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:05,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:05,866 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:05,867 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:05,867 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:05,874 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:06,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:08,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:08,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:16:10,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:13,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:13,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:13,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:14,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:14,008 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:14,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:14,019 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:14,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:16,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:16,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:18,496 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:18,497 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:18,497 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:18,510 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:18,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:20,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:21,511 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:25,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:25,967 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:25,967 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:25,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:26,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:26,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:28,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:28,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:28,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:28,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:32,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:34,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:34,459 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:34,460 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:34,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:34,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:36,648 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:38,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:38,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:16:39,936 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:39,937 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:39,938 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:39,948 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:40,652 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:42,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:43,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:43,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:43,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:43,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:44,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:44,621 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:44,622 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:44,631 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:44,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:46,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:48,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:49,750 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:49,751 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:49,751 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:49,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:50,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:50,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:53,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:54,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:54,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:54,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:54,449 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:54,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:56,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:58,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:58,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:58,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:58,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:58,705 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:58,706 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:58,714 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:59,699 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:59,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:00,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:04,750 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:05,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:05,491 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:05,492 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:05,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:05,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:06,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:08,947 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:17:09,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:10,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:10,224 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:10,224 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:10,235 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:10,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:12,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:13,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:13,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:13,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:14,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:14,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:14,926 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:14,940 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:14,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:15,730 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:16,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:19,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:21,091 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:21,092 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:21,092 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:21,101 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:21,743 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:22,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:25,102 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:25,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:25,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:25,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:25,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:25,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:26,754 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:28,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:28,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:28,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:30,292 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:30,292 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:30,293 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:30,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:30,300 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:30,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:32,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:34,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:34,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:34,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:34,719 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:34,766 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:35,720 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:36,771 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:38,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:17:39,568 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:39,569 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:39,569 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:39,580 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:39,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:40,780 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:41,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:43,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:43,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:43,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:44,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:44,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:44,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:44,895 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:45,786 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:46,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:46,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:49,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:49,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:49,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:49,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:49,796 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:50,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:52,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:54,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:54,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:54,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:54,176 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:54,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:56,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:57,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:58,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:58,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:58,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:58,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:58,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:58,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:58,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:59,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:00,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:02,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:04,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:04,133 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:04,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:04,142 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:04,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:06,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:08,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:08,951 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:18:08,979 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:08,980 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:08,980 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:08,987 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:09,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:10,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:13,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:13,256 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:13,256 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:13,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:13,263 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:13,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:13,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:13,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:13,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:14,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:18,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:19,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:19,023 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:19,023 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:19,029 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:19,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:20,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:24,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:24,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:24,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:24,568 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:24,575 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:24,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:26,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:28,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:28,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:28,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:29,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:29,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:29,753 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:29,753 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:29,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:29,864 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:30,867 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:34,798 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:35,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:35,560 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:35,560 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:35,568 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:35,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:36,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:38,954 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:18:39,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:42,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:42,234 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:42,235 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:42,241 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:42,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:43,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:43,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:43,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:44,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:45,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:47,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:47,198 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:47,199 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:47,205 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:47,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:48,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:51,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:52,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:52,080 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:52,080 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:52,087 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:52,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:54,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:57,087 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:58,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:58,239 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:58,239 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:58,245 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:58,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:58,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:58,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:58,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:00,908 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:02,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:03,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:03,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:03,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:03,128 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:03,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:04,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:07,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:07,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:07,651 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:07,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:07,657 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:07,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:08,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:08,955 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:19:12,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:13,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:13,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:13,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:16,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:16,036 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:16,037 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:16,042 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:16,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:18,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:18,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:22,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:22,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:22,779 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:22,784 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:22,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:23,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:24,938 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:28,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:28,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:28,095 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:28,101 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:28,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:28,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:28,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:28,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:29,563 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:30,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:32,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:32,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:32,024 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:32,030 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:32,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:34,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:35,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:37,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:37,538 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:37,538 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:37,544 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:37,953 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:38,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:38,961 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:19:40,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:42,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:42,990 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:42,991 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:42,996 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:43,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:43,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:43,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:43,961 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:44,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:46,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:49,040 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:49,041 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:49,041 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:49,046 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:49,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:50,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:52,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:55,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:55,385 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:55,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:55,391 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:55,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:56,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:57,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:58,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:58,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:58,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:02,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:03,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:03,829 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:03,829 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:03,835 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:03,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:04,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:07,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:08,798 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:08,799 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:08,799 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:08,805 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:08,960 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:20:08,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:10,992 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:12,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:13,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:13,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:13,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:14,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:14,134 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:14,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:14,139 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:14,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:16,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:18,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:20,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:20,850 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:20,850 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:20,856 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:21,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:23,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:23,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:25,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:25,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:25,498 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:25,504 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:26,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:27,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:28,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:28,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:28,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:29,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:33,906 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:33,906 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:33,907 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:33,912 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:34,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:34,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:35,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:38,962 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:20:39,031 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:39,031 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:39,032 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:39,037 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:40,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:40,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:41,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:43,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:43,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:43,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:45,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:45,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:45,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:45,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:45,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:46,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:47,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:50,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:52,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:52,031 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:52,031 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:52,037 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:53,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:53,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:56,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:58,076 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:58,077 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:58,077 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:58,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:58,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:58,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:58,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:59,042 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:59,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:01,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:02,707 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:02,708 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:02,708 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:02,715 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:03,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:05,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:06,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:07,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:07,672 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:07,672 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:07,678 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:08,051 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:08,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:21:09,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:11,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:13,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:13,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:13,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:14,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:14,927 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:14,927 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:14,933 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:15,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:16,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:17,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:19,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:19,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:19,475 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:19,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:20,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:21,069 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:22,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:24,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:24,258 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:24,258 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:24,264 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:25,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:27,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:28,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:28,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:28,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:28,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:30,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:30,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:30,743 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:30,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:31,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:33,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:33,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:35,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:35,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:35,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:35,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:36,084 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:37,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:38,966 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:21:38,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:43,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:43,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:43,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:43,851 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:43,852 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:43,852 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:43,857 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:44,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:44,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:45,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:48,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:48,934 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:48,934 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:48,940 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:49,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:49,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:51,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:53,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:53,389 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:53,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:53,394 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:54,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:55,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:55,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:58,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:58,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:58,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:00,596 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:01,867 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:01,867 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:01,867 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:01,873 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:02,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:03,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:05,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:06,499 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:06,499 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:06,499 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:06,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:07,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:08,972 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:22:09,120 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:10,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:11,765 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:11,765 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:11,765 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:11,771 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:12,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:13,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:13,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:13,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:13,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:16,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:16,296 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:16,296 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:16,296 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:16,302 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:17,129 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:19,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:21,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:22,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:22,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:22,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:22,349 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:23,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:25,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:26,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:28,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:28,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:28,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:30,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:30,551 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:30,552 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:30,557 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:31,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:31,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:33,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:36,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:38,971 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:22:39,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:39,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:39,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:39,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:39,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:41,155 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:42,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:43,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:43,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:43,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:43,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:43,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:43,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:43,648 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:44,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:45,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:47,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:47,750 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:47,751 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:47,751 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:47,757 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:48,162 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:49,165 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:51,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:51,994 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:51,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:51,999 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:52,167 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:53,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:53,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:56,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:56,270 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:56,270 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:56,275 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:57,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:57,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:58,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:58,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:58,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:58,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:03,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:03,429 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:03,429 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:03,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:03,435 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:04,178 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:05,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:08,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:08,116 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:08,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:08,121 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:08,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:08,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:23:08,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:09,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:12,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:12,646 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:12,646 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:12,652 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:13,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:13,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:13,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:13,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:14,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:15,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:17,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:17,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:17,787 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:17,793 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:18,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:19,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:19,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:22,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:22,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:22,466 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:22,472 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:23,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:25,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:25,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:27,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:27,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:27,224 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:27,230 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:28,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:28,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:28,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:28,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:29,209 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:30,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:32,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:32,350 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:32,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:32,356 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:33,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:35,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:36,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:37,767 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:37,768 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:37,768 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:37,774 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:38,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:38,975 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:23:39,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:41,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:43,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:43,174 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:43,175 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:43,180 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:43,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:43,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:43,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:43,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:45,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:47,415 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:47,416 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:47,416 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:47,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:47,422 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:48,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:49,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:52,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:52,397 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:52,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:52,404 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:53,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:53,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:55,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:57,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:57,318 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:57,319 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:57,325 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:58,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:58,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:58,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:58,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:58,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:59,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:02,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:02,116 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:02,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:02,122 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:02,246 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:03,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:04,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:07,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:07,599 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:07,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:07,604 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:08,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:08,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:24:09,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:09,979 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:13,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:13,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:13,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:14,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:14,737 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:14,738 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:14,743 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:15,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:15,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:17,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:19,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:19,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:19,439 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:19,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:20,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:21,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:21,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:24,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:24,316 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:24,316 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:24,321 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:25,276 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:25,276 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:27,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:28,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:28,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:28,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:29,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:29,661 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:29,662 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:29,668 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:30,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:31,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:32,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:34,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:34,118 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:34,118 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:34,124 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:34,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:35,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:38,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:38,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:38,754 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:38,754 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:38,760 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:38,980 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:24:39,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:41,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:43,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:43,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:43,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:43,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:44,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:44,632 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:44,632 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:44,638 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:45,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:47,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:48,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:49,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:49,963 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:49,963 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:49,969 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:50,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:51,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:53,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:54,508 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:54,509 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:54,509 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:54,515 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:55,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:57,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:58,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:58,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:58,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:59,622 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:00,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:00,290 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:00,290 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:00,297 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:00,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:01,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:04,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:04,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:04,202 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:04,208 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:04,320 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:05,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:05,321 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:08,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:08,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:08,894 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:08,899 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:08,982 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:25:09,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:10,984 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:11,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:13,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:13,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:13,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:13,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:13,599 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:13,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:13,604 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:14,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:15,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:16,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:18,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:18,374 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:18,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:18,379 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:19,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:19,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:22,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:24,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:24,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:24,066 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:24,071 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:24,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:25,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:28,072 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:28,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:28,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:28,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:30,903 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:30,904 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:30,904 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:30,910 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:31,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:33,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:33,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:35,595 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:35,599 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:35,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:35,604 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:36,353 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:37,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:38,984 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:25:38,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:43,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:43,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:43,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:44,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:45,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:45,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:45,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:45,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:45,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:47,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:50,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:50,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:50,371 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:50,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:50,377 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:51,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:51,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:55,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:57,952 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:57,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:57,953 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:57,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:58,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:58,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:58,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:58,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:59,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:00,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:04,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:04,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:04,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:04,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:04,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:05,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:06,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:08,987 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:26:09,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:09,268 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:09,268 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:09,274 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:09,386 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:11,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:11,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:13,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:13,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:13,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:14,776 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:14,777 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:14,777 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:14,783 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:15,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:16,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:17,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:20,048 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:20,049 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:20,049 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:20,054 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:20,400 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:21,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:22,055 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:27,055 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:27,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:27,369 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:27,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:27,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:27,407 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:28,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:28,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:28,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:29,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:32,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:32,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:32,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:32,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:32,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:32,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:33,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:36,958 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:36,959 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:36,959 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:36,965 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:37,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:37,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:38,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:26:39,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:42,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:42,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:42,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:42,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:43,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:43,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:43,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:43,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:43,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:43,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:48,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:48,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:48,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:48,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:48,733 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:49,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:51,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:53,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:53,236 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:53,236 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:53,242 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:53,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:54,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:55,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:58,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:58,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:58,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:58,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:58,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:58,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:58,642 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:59,442 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:59,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:01,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:03,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:03,189 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:03,190 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:03,195 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:03,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:05,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:05,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:08,991 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:27:10,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:11,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:11,378 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:11,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:11,384 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:11,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:13,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:13,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:13,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:13,458 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:16,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:18,875 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:18,875 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:18,875 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:18,881 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:19,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:21,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:21,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:24,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:24,222 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:24,223 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:24,233 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:24,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:25,474 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:27,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:28,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:28,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:28,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:30,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:30,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:30,066 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:30,072 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:30,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:31,480 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:33,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:36,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:36,125 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:36,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:36,131 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:36,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:37,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:38,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:38,993 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:27:42,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:42,355 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:42,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:42,360 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:42,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:43,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:43,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:43,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:43,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:43,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:46,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:46,511 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:46,511 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:46,520 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:47,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:47,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:49,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:51,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:51,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:51,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:51,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:51,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:53,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:55,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:56,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:56,051 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:56,051 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:56,057 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:56,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:57,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:58,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:58,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:58,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:00,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:01,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:01,554 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:01,554 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:01,560 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:02,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:03,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:05,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:06,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:06,421 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:06,421 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:06,427 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:06,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:07,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:08,995 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:28:10,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:13,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:13,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:13,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:14,768 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:14,769 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:14,769 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:14,775 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:15,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:16,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:17,530 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:20,076 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:20,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:20,077 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:20,083 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:20,533 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:21,536 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:22,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:24,767 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:24,768 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:24,768 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:24,773 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:25,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:27,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:27,774 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:28,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:28,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:28,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:28,867 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:28,868 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:28,868 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:28,874 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:29,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:31,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:32,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:33,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:33,670 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:33,670 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:33,676 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:34,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:35,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:37,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:37,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:37,702 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:37,708 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:38,554 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:38,708 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:38,998 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:28:39,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:43,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:43,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:43,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:44,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:44,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:44,616 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:44,616 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:44,622 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:45,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:45,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:49,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:50,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:50,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:50,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:50,016 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:50,567 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:51,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:55,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:56,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:56,444 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:56,444 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:56,451 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:56,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:57,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:58,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:58,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:58,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:00,632 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:04,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:04,967 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:04,967 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:04,972 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:05,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:06,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:07,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:08,964 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:08,965 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:08,965 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:08,970 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:09,003 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:29:09,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:11,588 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:12,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:13,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:13,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:13,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:13,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:13,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:13,619 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:13,625 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:14,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:15,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:17,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:20,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:20,025 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:20,026 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:20,031 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:20,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:21,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:23,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:24,486 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:24,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:24,487 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:24,493 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:24,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:25,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:28,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:28,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:28,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:28,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:28,944 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:28,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:28,945 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:28,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:29,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:31,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:33,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:33,644 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:33,644 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:33,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:33,650 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:34,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:35,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:38,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:38,175 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:38,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:38,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:38,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:39,002 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:29:39,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:39,624 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:43,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:43,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:43,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:43,627 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:43,628 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:43,634 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:44,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:44,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:45,630 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:47,299 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:47,300 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:47,300 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:47,306 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:47,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:49,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:50,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:52,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:52,503 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:52,503 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:52,509 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:52,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:53,640 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:55,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:57,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:57,873 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:57,874 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:57,879 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:58,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:58,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:58,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:58,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:59,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:00,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:04,950 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:04,951 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:04,951 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:04,957 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:05,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:05,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:07,655 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:09,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:30:09,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:09,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:09,414 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:09,420 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:09,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:11,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:11,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:13,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:13,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:13,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:16,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:17,676 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:17,677 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:17,677 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:17,683 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:18,666 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:19,669 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:21,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:21,772 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:21,773 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:21,773 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:21,779 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:22,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:23,674 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:26,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:26,310 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:26,310 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:26,316 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:26,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:27,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:27,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:28,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:28,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:28,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:32,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:32,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:32,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:32,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:32,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:32,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:33,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:37,686 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:39,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:30:41,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:41,166 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:41,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:41,172 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:41,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:43,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:43,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:43,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:43,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:43,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:46,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:46,464 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:46,464 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:46,470 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:46,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:47,701 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:48,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:50,904 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:50,905 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:50,906 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:50,911 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:51,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:53,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:53,912 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:56,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:56,510 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:56,510 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:56,516 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:56,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:57,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:58,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:58,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:58,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:59,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:02,539 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:02,539 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:02,540 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:02,546 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:02,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:03,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:05,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:09,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:31:09,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:09,215 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:09,215 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:09,221 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:09,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:11,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:11,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:13,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:13,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:13,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:13,779 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:13,780 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:13,780 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:13,786 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:14,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:15,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:16,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:18,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:18,557 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:18,557 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:18,563 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:18,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:19,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:22,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:23,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:23,306 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:23,307 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:23,313 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:23,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:25,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:28,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:28,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:28,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:28,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:30,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:30,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:30,307 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:30,313 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:30,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:31,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:33,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:38,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:38,754 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:38,755 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:38,755 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:38,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:39,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:31:39,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:39,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:43,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:43,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:43,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:43,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:44,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:44,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:44,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:44,648 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:44,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:45,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:48,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:49,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:49,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:49,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:49,293 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:49,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:51,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:54,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:55,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:55,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:55,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:55,717 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:55,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:57,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:58,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:58,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:58,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:59,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:02,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:02,979 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:02,979 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:02,985 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:03,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:05,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:05,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:07,936 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:07,937 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:07,937 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:07,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:08,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:09,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:32:09,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:11,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:12,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:12,627 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:12,628 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:12,634 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:12,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:13,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:13,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:13,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:13,796 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:16,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:17,867 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:17,868 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:17,868 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:17,874 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:18,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:19,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:21,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:22,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:22,638 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:22,638 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:22,644 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:22,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:23,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:27,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:28,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:28,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:28,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:28,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:28,589 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:28,590 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:28,596 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:28,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:29,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:33,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:37,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:37,039 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:37,040 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:37,045 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:37,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:39,018 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:32:39,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:39,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:42,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:42,531 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:42,531 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:42,537 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:42,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:43,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:43,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:43,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:43,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:44,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:48,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:48,742 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:48,742 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:48,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:48,834 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:49,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:49,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:52,667 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:52,668 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:52,669 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:52,675 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:52,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:53,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:55,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:57,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:57,189 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:57,190 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:57,196 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:57,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:58,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:58,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:58,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:59,849 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:01,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:01,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:01,693 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:01,694 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:01,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:01,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:03,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:06,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:06,274 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:06,275 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:06,280 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:06,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:07,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:07,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:09,017 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:33:11,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:11,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:11,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:11,185 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:11,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:13,185 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:13,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:13,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:13,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:13,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:17,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:17,571 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:17,571 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:17,578 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:17,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:18,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:19,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:22,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:22,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:22,448 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:22,454 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:22,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:23,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:24,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:28,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:28,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:28,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:28,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:28,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:28,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:28,930 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:29,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:29,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:29,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:33,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:33,993 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:33,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:33,999 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:34,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:35,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:35,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:38,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:38,839 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:38,839 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:38,845 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:38,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:39,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:33:39,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:40,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:43,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:43,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:43,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:43,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:43,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:43,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:43,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:43,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:45,736 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:45,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:47,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:47,741 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:47,741 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:47,747 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:47,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:49,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:50,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:52,859 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:52,860 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:52,860 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:52,866 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:52,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:53,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:55,867 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:58,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:58,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:58,288 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:58,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:58,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:58,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:58,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:58,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:59,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:01,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:06,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:06,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:06,658 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:06,658 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:06,665 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:06,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:07,928 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:09,023 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:34:11,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:11,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:11,790 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:11,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:11,796 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:11,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:13,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:13,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:13,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:13,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:16,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:16,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:16,033 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:16,040 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:16,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:17,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:17,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:20,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:20,583 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:20,583 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:20,589 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:20,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:21,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:22,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:25,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:25,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:25,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:25,304 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:25,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:27,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:28,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:28,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:28,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:28,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:29,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:29,846 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:29,846 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:29,852 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:29,953 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:31,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:33,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:34,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:34,316 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:34,316 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:34,322 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:34,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:35,962 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:39,028 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:34:39,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:39,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:39,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:39,039 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:39,045 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:39,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:39,967 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:43,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:43,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:43,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:43,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:43,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:43,355 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:43,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:43,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:44,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:45,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:49,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:51,894 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:51,895 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:51,895 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:51,902 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:51,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:53,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:54,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:56,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:56,136 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:56,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:56,142 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:56,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:57,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:58,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:58,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:58,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:00,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:00,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:00,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:00,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:00,716 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:00,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:01,992 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:05,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:05,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:05,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:05,184 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:05,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:06,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:07,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:09,028 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:35:10,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:10,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:10,414 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:10,420 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:11,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:11,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:12,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:13,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:13,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:13,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:15,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:15,802 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:15,803 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:15,812 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:16,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:16,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:18,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:21,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:22,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:22,640 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:22,641 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:22,650 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:23,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:24,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:27,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:27,282 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:27,282 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:27,283 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:27,290 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:28,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:28,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:28,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:28,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:30,025 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:31,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:31,598 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:31,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:31,604 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:32,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:32,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:34,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:37,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:38,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:38,410 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:38,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:38,416 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:39,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:35:39,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:40,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:43,031 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:43,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:43,124 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:43,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:43,130 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:43,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:43,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:43,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:44,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:44,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:48,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:51,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:51,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:51,415 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:51,420 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:52,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:54,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:54,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:57,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:57,158 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:57,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:57,164 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:58,051 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:58,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:58,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:58,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:59,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:00,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:04,084 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:04,085 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:04,085 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:04,091 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:05,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:05,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:06,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:09,032 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:36:10,504 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:10,505 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:10,505 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:10,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:10,511 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:11,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:12,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:13,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:13,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:13,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:15,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:15,820 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:15,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:15,820 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:15,826 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:16,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:18,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:20,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:21,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:21,283 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:21,283 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:21,289 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:22,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:24,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:25,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:25,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:25,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:25,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:26,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:26,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:28,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:28,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:28,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:28,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:30,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:30,990 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:30,990 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:30,996 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:31,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:32,033 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:32,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:36,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:36,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:36,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:36,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:37,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:37,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:38,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:39,034 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:36:43,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:43,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:43,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:43,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:45,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:45,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:45,499 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:45,504 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:46,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:48,109 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:48,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:50,076 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:50,077 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:50,077 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:50,083 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:50,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:52,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:54,084 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:56,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:56,393 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:56,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:56,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:57,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:58,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:58,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:58,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:58,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:59,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:01,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:01,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:01,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:01,923 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:02,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:04,128 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:04,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:06,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:06,422 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:06,422 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:06,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:07,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:08,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:09,039 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:37:10,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:10,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:10,410 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:10,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:10,421 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:11,135 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:12,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:13,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:13,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:13,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:15,639 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:15,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:15,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:15,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:15,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:16,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:18,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:20,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:20,379 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:20,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:20,385 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:21,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:21,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:22,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:25,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:25,046 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:25,046 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:25,053 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:25,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:26,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:27,054 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:28,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:28,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:28,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:29,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:29,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:29,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:29,896 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:30,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:32,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:32,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:34,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:34,466 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:34,467 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:34,472 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:35,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:36,169 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:38,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:39,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:37:41,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:41,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:41,522 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:41,527 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:42,173 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:43,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:43,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:43,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:43,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:44,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:46,460 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:46,460 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:46,461 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:46,467 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:47,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:48,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:49,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:51,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:51,631 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:51,631 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:51,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:52,182 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:54,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:54,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:56,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:56,887 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:56,887 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:56,893 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:57,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:58,192 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:58,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:58,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:58,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:00,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:01,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:01,271 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:01,272 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:01,278 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:02,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:04,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:06,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:07,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:07,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:07,654 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:07,660 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:08,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:09,043 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:38:10,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:12,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:12,796 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:12,796 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:12,797 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:12,802 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:13,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:13,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:13,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:13,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:14,210 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:17,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:17,368 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:17,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:17,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:17,374 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:18,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:20,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:22,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:26,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:26,208 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:26,208 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:26,214 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:26,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:28,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:28,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:28,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:28,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:31,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:31,304 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:31,304 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:31,310 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:32,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:33,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:34,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:36,240 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:36,241 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:36,241 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:36,247 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:37,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:38,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:39,043 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:38:39,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:40,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:40,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:40,767 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:40,773 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:41,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:42,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:43,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:43,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:43,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:44,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:45,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:45,630 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:45,630 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:45,637 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:46,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:48,250 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:49,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:50,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:50,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:50,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:50,763 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:51,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:52,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:54,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:55,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:55,358 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:55,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:55,364 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:56,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:58,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:58,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:58,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:58,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:00,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:03,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:03,620 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:03,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:03,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:04,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:05,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:06,267 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:09,045 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:39:09,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:09,962 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:09,962 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:09,967 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:10,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:10,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:12,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:13,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:13,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:13,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:14,739 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:14,739 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:14,740 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:14,745 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:15,276 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:16,279 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:16,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:19,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:19,893 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:19,893 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:19,899 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:20,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:21,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:22,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:25,273 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:25,274 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:25,274 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:25,280 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:25,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:26,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:27,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:28,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:28,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:30,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:30,113 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:30,114 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:30,125 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:30,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:32,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:33,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:35,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:35,648 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:35,648 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:35,653 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:36,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:38,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:38,654 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:39,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:39:40,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:40,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:40,038 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:40,044 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:40,303 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:42,307 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:43,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:43,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:43,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:44,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:44,352 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:44,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:44,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:44,357 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:45,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:46,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:48,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:48,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:48,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:48,870 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:49,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:49,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:50,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:54,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:54,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:54,999 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:54,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:55,005 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:55,321 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:56,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:58,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:58,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:58,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:00,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:01,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:01,801 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:01,801 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:01,807 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:02,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:04,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:05,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:06,942 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:06,942 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:06,943 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:06,948 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:07,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:08,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:09,049 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:40:11,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:11,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:11,311 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:11,311 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:11,317 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:11,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:12,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:13,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:13,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:13,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:15,827 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:15,827 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:15,827 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:15,833 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:16,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:16,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:18,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:21,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:21,331 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:21,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:21,337 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:21,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:22,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:22,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:26,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:26,682 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:26,683 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:26,688 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:27,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:27,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:28,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:28,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:28,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:31,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:31,350 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:31,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:31,356 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:31,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:32,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:33,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:35,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:35,638 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:35,639 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:35,645 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:36,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:38,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:38,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:39,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:40:43,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:43,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:43,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:44,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:44,045 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:44,045 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:44,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:44,051 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:44,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:46,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:49,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:50,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:50,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:50,650 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:50,656 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:51,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:52,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:54,656 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:58,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:58,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:58,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:59,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:59,358 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:59,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:59,365 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:59,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:00,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:00,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:03,820 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:03,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:03,821 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:03,827 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:04,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:05,828 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:06,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:08,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:08,400 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:08,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:08,407 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:09,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:41:09,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:10,402 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:11,055 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:12,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:12,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:12,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:12,293 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:12,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:13,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:13,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:13,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:14,407 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:16,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:20,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:20,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:20,953 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:20,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:21,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:21,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:22,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:26,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:27,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:27,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:27,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:27,170 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:27,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:28,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:28,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:28,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:31,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:31,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:31,612 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:31,618 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:32,427 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:32,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:34,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:36,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:36,954 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:36,954 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:36,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:37,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:37,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:38,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:39,059 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:41:43,060 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:43,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:43,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:43,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:45,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:45,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:45,482 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:45,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:46,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:46,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:48,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:50,550 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:50,551 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:50,551 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:50,557 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:51,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:52,450 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:53,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:56,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:56,810 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:56,810 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:56,815 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:57,454 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:58,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:58,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:58,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:58,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:58,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:01,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:01,344 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:01,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:01,350 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:01,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:02,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:04,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:06,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:06,594 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:06,594 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:06,600 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:07,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:08,465 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:09,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:42:10,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:13,163 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:13,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:13,169 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:13,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:13,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:13,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:13,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:14,474 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:15,622 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:18,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:18,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:18,664 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:18,670 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:19,480 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:20,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:20,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:23,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:23,128 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:23,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:23,133 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:23,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:24,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:26,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:27,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:27,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:27,499 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:27,504 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:28,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:28,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:28,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:31,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:35,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:35,830 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:35,830 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:35,835 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:36,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:36,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:38,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:39,060 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:42:40,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:40,151 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:40,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:40,157 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:40,505 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:42,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:42,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:43,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:43,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:43,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:45,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:45,248 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:45,249 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:45,254 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:45,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:46,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:47,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:49,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:49,620 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:49,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:49,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:50,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:52,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:52,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:54,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:54,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:54,280 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:54,285 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:54,521 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:56,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:58,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:58,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:58,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:58,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:59,754 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:59,754 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:59,755 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:59,760 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:00,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:02,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:03,797 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:04,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:04,383 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:04,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:04,389 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:04,533 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:06,536 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:09,063 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:43:09,064 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:09,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:09,477 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:09,477 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:09,483 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:09,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:10,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:13,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:13,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:13,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:14,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:14,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:14,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:14,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:14,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:15,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:16,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:19,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:21,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:21,225 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:21,225 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:21,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:21,554 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:22,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:25,232 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:27,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:27,400 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:27,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:27,406 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:27,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:28,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:28,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:30,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:33,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:33,174 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:33,174 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:33,180 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:33,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:34,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:36,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:37,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:37,543 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:37,543 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:37,548 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:37,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:38,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:39,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:43:42,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:42,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:42,511 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:42,511 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:42,517 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:42,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:43,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:43,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:43,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:44,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:47,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:48,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:48,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:48,494 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:48,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:48,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:50,589 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:53,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:55,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:55,308 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:55,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:55,314 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:55,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:56,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:58,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:58,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:58,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:58,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:59,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:59,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:59,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:59,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:00,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:02,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:03,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:07,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:07,062 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:07,062 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:07,068 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:07,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:08,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:09,068 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:09,070 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:44:13,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:13,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:13,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:13,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:13,637 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:13,638 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:13,643 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:14,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:14,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:14,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:17,879 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:17,880 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:17,880 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:17,886 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:18,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:19,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:20,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:24,503 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:24,504 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:24,504 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:24,510 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:24,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:25,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:26,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:28,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:28,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:29,811 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:29,811 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:29,812 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:29,817 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:30,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:30,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:32,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:35,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:36,468 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:36,468 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:36,469 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:36,474 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:36,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:38,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:39,069 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:44:41,070 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:43,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:43,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:43,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:43,261 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:43,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:43,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:43,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:43,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:44,650 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:46,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:51,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:52,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:52,445 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:52,445 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:52,452 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:52,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:54,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:57,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:57,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:57,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:57,643 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:57,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:57,663 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:58,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:58,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:58,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:58,666 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:02,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:03,283 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:03,284 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:03,284 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:03,290 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:03,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:04,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:08,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:08,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:08,600 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:08,600 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:08,605 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:08,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:09,074 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:45:10,680 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:13,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:13,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:13,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:13,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:14,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:14,306 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:14,306 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:14,312 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:14,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:16,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:19,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:19,198 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:19,198 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:19,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:19,204 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:19,690 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:20,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:23,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:23,346 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:23,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:23,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:23,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:24,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:24,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:27,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:27,620 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:27,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:27,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:27,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:28,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:28,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:28,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:28,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:29,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:32,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:32,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:32,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:32,930 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:33,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:34,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:34,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:38,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:38,419 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:38,419 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:38,425 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:38,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:39,074 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:45:40,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:40,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:43,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:43,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:43,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:44,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:44,577 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:44,577 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:44,582 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:44,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:45,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:46,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:50,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:53,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:53,016 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:53,017 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:53,022 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:53,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:54,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:56,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:57,920 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:57,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:57,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:57,926 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:58,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:58,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:58,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:58,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:00,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:01,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:03,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:03,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:03,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:03,337 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:03,741 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:04,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:07,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:07,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:07,785 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:07,785 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:07,794 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:08,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:08,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:09,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:46:12,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:12,399 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:12,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:12,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:12,405 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:12,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:13,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:13,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:13,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:14,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:17,544 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:20,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:20,705 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:20,705 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:20,711 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:20,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:22,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:22,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:25,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:25,495 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:25,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:25,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:25,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:26,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:28,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:28,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:28,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:28,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:29,939 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:29,940 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:29,940 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:29,946 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:30,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:32,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:33,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:35,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:35,240 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:35,240 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:35,246 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:35,780 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:36,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:39,078 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:46:39,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:40,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:40,457 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:40,457 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:40,463 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:40,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:42,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:43,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:43,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:43,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:44,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:44,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:44,986 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:44,986 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:44,992 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:45,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:46,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:49,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:49,531 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:49,531 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:49,536 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:49,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:50,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:50,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:55,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:55,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:55,662 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:55,662 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:55,667 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:55,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:56,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:58,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:58,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:58,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:00,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:00,334 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:00,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:00,340 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:00,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:01,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:02,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:06,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:08,879 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:08,880 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:08,880 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:08,885 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:09,081 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:47:09,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:10,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:12,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:13,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:13,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:13,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:13,964 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:13,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:13,965 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:13,970 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:14,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:16,831 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:18,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:18,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:18,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:18,488 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:18,493 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:18,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:20,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:23,068 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:23,069 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:23,069 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:23,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:23,075 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:23,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:24,844 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:27,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:27,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:27,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:27,648 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:27,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:28,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:28,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:28,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:28,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:28,849 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:32,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:32,354 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:32,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:32,360 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:32,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:34,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:34,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:38,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:38,433 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:38,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:38,439 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:38,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:39,086 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:47:40,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:40,863 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:43,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:43,214 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:43,214 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:43,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:43,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:43,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:43,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:43,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:44,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:45,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:48,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:48,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:48,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:48,028 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:48,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:50,875 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:51,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:53,782 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:53,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:53,783 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:53,789 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:53,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:54,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:56,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:58,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:58,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:58,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:00,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:00,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:00,058 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:00,063 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:00,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:02,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:02,889 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:07,102 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:08,507 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:08,508 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:08,508 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:08,513 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:08,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:09,085 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:48:10,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:12,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:12,938 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:12,938 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:12,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:12,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:13,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:13,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:13,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:13,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:14,902 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:17,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:17,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:17,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:17,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:17,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:18,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:18,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:21,884 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:21,885 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:21,885 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:21,891 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:21,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:22,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:23,891 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:26,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:26,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:26,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:26,017 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:26,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:28,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:28,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:28,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:28,924 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:29,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:31,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:31,265 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:31,266 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:31,272 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:31,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:32,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:35,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:35,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:35,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:35,498 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:35,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:35,932 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:36,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:39,090 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:48:40,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:40,336 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:40,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:40,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:40,342 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:40,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:42,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:43,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:43,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:43,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:45,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:45,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:45,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:45,060 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:45,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:46,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:46,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:49,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:49,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:49,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:49,715 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:49,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:50,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:51,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:55,291 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:55,292 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:55,292 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:55,298 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:55,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:56,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:57,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:58,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:58,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:58,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:01,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:01,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:01,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:01,216 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:01,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:02,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:03,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:05,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:05,629 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:05,630 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:05,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:05,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:06,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:08,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:09,090 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:49:11,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:11,997 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:11,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:12,003 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:12,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:12,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:13,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:13,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:13,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:17,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:17,398 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:17,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:17,404 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:17,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:18,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:19,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:23,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:23,326 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:23,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:23,332 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:23,984 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:24,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:25,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:27,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:27,606 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:27,606 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:27,612 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:27,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:28,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:28,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:28,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:30,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:31,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:31,651 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:31,651 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:31,657 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:31,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:32,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:35,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:36,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:36,927 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:36,927 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:36,933 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:36,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:39,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:39,095 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:49:41,096 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:41,431 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:41,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:41,432 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:41,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:42,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:43,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:43,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:43,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:43,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:46,654 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:47,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:47,590 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:47,590 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:47,596 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:48,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:49,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:52,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:53,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:53,289 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:53,289 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:53,295 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:54,019 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:55,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:58,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:58,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:58,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:58,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:00,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:00,276 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:00,276 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:00,282 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:01,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:03,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:03,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:04,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:04,939 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:04,939 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:04,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:05,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:07,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:08,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:09,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:50:09,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:09,115 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:09,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:09,121 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:10,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:11,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:13,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:13,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:14,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:14,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:14,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:14,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:14,216 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:15,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:17,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:19,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:20,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:20,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:20,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:20,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:20,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:21,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:24,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:24,911 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:24,911 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:24,911 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:24,917 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:25,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:27,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:28,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:28,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:30,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:32,078 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:32,079 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:32,079 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:32,084 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:33,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:33,069 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:36,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:36,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:36,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:36,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:36,349 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:37,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:39,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:39,096 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:50:41,231 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:41,232 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:41,232 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:41,232 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:41,238 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:42,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:43,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:43,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:43,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:43,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:45,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:45,863 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:45,863 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:45,869 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:46,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:46,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:47,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:51,906 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:54,750 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:54,750 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:54,751 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:54,757 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:55,092 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:57,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:57,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:58,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:58,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:58,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:59,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:59,594 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:59,594 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:59,600 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:00,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:01,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:03,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:04,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:04,266 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:04,266 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:04,272 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:05,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:07,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:09,101 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:51:09,102 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:13,071 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:13,072 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:13,073 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:13,078 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:13,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:13,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:13,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:14,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:15,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:19,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:22,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:22,354 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:22,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:22,361 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:23,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:25,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:25,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:26,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:26,617 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:26,618 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:26,623 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:27,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:28,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:28,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:29,131 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:30,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:30,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:30,982 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:30,982 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:30,988 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:31,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:33,137 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:35,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:35,682 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:35,682 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:35,683 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:35,688 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:36,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:37,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:39,100 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:51:41,102 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:43,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:43,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:43,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:44,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:44,083 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:44,084 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:44,089 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:44,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:45,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:46,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:49,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:49,029 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:49,029 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:49,035 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:49,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:51,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:52,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:55,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:55,361 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:55,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:55,367 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:56,162 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:57,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:57,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:58,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:58,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:58,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:00,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:00,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:00,020 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:00,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:00,167 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:01,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:03,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:05,273 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:05,274 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:05,274 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:05,279 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:06,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:07,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:08,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:09,106 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:52:10,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:10,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:10,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:10,935 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:11,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:13,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:13,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:13,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:13,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:15,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:15,156 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:15,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:15,162 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:15,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:17,187 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:19,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:20,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:20,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:20,515 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:20,522 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:21,192 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:23,196 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:24,523 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:24,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:24,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:24,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:24,924 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:25,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:27,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:28,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:28,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:29,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:29,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:29,742 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:29,742 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:29,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:30,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:31,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:34,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:37,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:37,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:37,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:37,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:37,210 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:39,105 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:52:39,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:40,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:41,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:41,704 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:41,705 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:41,710 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:42,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:43,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:43,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:43,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:43,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:45,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:49,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:49,139 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:49,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:49,145 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:49,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:51,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:51,225 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:54,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:54,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:54,299 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:54,304 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:55,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:56,304 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:57,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:58,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:58,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:58,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:59,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:59,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:59,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:59,244 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:00,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:01,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:02,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:07,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:07,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:07,716 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:07,717 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:07,722 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:08,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:09,107 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:53:09,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:12,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:12,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:12,475 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:12,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:12,480 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:13,246 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:13,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:13,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:15,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:16,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:16,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:16,772 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:16,778 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:17,251 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:17,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:19,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:22,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:23,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:23,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:23,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:23,170 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:23,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:25,264 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:27,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:27,584 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:27,584 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:27,590 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:28,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:28,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:28,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:29,267 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:32,532 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:32,532 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:32,532 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:32,538 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:33,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:34,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:35,274 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:37,047 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:37,048 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:37,048 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:37,053 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:37,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:39,109 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:53:39,279 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:40,111 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:41,902 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:41,903 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:41,903 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:41,909 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:42,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:43,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:43,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:43,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:43,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:45,546 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:46,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:46,352 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:46,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:46,358 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:47,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:49,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:50,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:50,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:50,889 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:50,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:50,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:51,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:53,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:55,895 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:56,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:56,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:56,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:56,356 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:57,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:58,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:58,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:58,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:59,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:01,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:01,384 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:01,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:01,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:01,391 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:02,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:03,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:06,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:06,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:06,952 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:06,952 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:06,957 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:07,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:09,111 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:54:09,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:12,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:13,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:14,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:14,239 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:14,239 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:14,245 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:14,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:15,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:17,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:18,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:18,967 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:18,967 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:18,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:19,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:21,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:22,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:23,718 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:23,718 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:23,719 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:23,724 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:24,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:25,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:28,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:28,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:28,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:28,346 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:28,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:28,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:29,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:29,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:33,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:33,158 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:33,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:33,164 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:33,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:34,204 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:35,349 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:37,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:37,630 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:37,630 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:37,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:38,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:39,114 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:54:39,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:40,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:43,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:43,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:43,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:43,342 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:43,355 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:43,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:43,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:43,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:45,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:45,606 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:47,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:47,648 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:47,648 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:47,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:48,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:49,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:50,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:52,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:52,565 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:52,565 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:52,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:53,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:55,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:56,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:57,102 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:57,103 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:57,103 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:57,109 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:57,372 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:58,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:58,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:58,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:59,376 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:01,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:02,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:02,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:02,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:02,320 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:02,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:03,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:06,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:06,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:06,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:06,376 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:06,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:07,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:07,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:09,116 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:55:13,118 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:13,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:14,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:14,118 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:14,118 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:14,124 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:14,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:15,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:18,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:19,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:19,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:19,878 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:19,885 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:20,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:21,400 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:23,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:24,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:24,200 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:24,200 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:24,205 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:24,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:25,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:28,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:28,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:29,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:29,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:29,688 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:29,688 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:29,694 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:30,407 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:31,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:34,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:34,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:34,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:34,898 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:34,904 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:35,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:37,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:39,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:55:39,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:39,327 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:39,327 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:39,333 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:39,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:40,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:41,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:43,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:43,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:43,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:45,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:47,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:47,833 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:47,834 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:47,839 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:48,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:49,428 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:50,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:54,550 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:54,551 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:54,551 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:54,557 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:55,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:56,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:57,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:58,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:58,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:58,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:00,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:00,302 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:00,302 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:00,308 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:00,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:01,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:02,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:07,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:08,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:08,996 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:08,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:09,002 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:09,121 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:56:09,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:11,450 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:13,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:13,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:13,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:13,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:13,668 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:13,668 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:13,673 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:14,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:15,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:18,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:18,338 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:18,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:18,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:18,345 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:18,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:19,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:23,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:26,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:26,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:26,243 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:26,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:26,469 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:27,472 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:28,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:28,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:28,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:32,712 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:32,713 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:32,713 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:32,719 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:33,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:33,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:35,480 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:38,754 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:39,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:56:40,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:40,174 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:40,174 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:40,179 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:40,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:41,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:43,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:43,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:43,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:44,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:44,935 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:44,935 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:44,936 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:44,941 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:45,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:47,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:49,732 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:49,733 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:49,733 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:49,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:49,739 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:50,496 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:51,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:54,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:55,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:55,001 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:55,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:55,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:55,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:57,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:58,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:58,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:58,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:59,543 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:59,544 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:59,544 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:59,550 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:00,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:00,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:01,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:05,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:05,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:05,227 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:05,233 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:05,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:06,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:07,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:09,125 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:57:09,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:09,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:09,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:09,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:10,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:11,521 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:11,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:13,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:13,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:13,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:15,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:15,218 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:15,218 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:15,224 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:15,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:17,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:17,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:22,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:22,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:22,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:22,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:22,530 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:23,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:23,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:26,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:26,701 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:26,701 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:26,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:27,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:28,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:28,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:28,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:28,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:29,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:31,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:31,039 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:31,039 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:31,045 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:31,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:33,545 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:34,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:36,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:36,244 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:36,244 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:36,250 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:36,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:37,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:39,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:57:39,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:40,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:40,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:40,950 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:40,956 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:41,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:43,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:43,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:43,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:43,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:44,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:47,816 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:47,817 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:47,817 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:47,823 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:48,560 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:49,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:49,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:52,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:52,834 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:52,834 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:52,840 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:53,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:55,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:55,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:58,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:58,234 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:58,235 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:58,241 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:58,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:58,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:58,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:58,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:59,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:01,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:03,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:03,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:03,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:03,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:03,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:05,581 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:07,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:09,129 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:58:09,930 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:09,931 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:09,931 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:09,937 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:10,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:11,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:12,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:13,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:13,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:13,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:14,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:14,046 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:14,046 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:14,053 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:14,589 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:15,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:18,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:21,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:21,495 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:21,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:21,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:21,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:23,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:23,600 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:27,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:27,375 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:27,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:27,381 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:27,603 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:28,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:28,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:28,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:28,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:29,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:32,296 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:32,297 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:32,297 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:32,303 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:32,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:33,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:34,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:36,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:36,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:36,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:36,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:37,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:39,131 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:58:39,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:40,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:43,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:43,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:43,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:45,462 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:45,462 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:45,463 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:45,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:45,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:45,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:47,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:50,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:50,357 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:50,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:50,363 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:50,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:51,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:51,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:54,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:54,638 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:54,638 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:54,644 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:55,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:55,640 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:56,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:58,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:58,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:58,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:59,988 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:59,989 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:59,989 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:59,995 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:00,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:01,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:02,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:05,485 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:05,486 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:05,486 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:05,492 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:05,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:07,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:07,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:09,136 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:59:09,715 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:09,716 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:09,716 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:09,722 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:10,655 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:11,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:12,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:13,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:13,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:13,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:14,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:14,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:14,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:14,339 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:14,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:15,663 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:18,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:19,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:19,522 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:19,522 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:19,528 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:19,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:21,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:23,528 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:27,866 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:27,867 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:27,867 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:27,873 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:28,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:28,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:28,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:28,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:28,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:29,682 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:33,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:33,376 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:33,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:33,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:33,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:34,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:35,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:37,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:37,706 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:37,706 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:37,712 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:38,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:39,135 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:59:39,692 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:40,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:43,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:43,213 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:43,213 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:43,218 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:43,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:43,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:43,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:43,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:45,639 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:45,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:49,024 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:49,025 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:49,025 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:49,031 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:49,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:51,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:51,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:53,944 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:53,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:53,945 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:53,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:54,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:55,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:56,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:57,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:57,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:57,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:57,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:58,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:58,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:58,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:58,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:59,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:02,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:02,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:02,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:02,883 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:02,889 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:03,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:05,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:07,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:09,140 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:00:09,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:09,635 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:09,635 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:09,641 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:09,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:11,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:13,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:13,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:13,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:13,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:14,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:14,923 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:14,923 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:14,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:15,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:17,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:18,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:19,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:19,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:19,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:19,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:19,738 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:21,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:24,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:25,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:25,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:25,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:25,527 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:25,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:27,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:28,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:28,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:28,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:29,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:31,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:31,074 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:31,074 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:31,080 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:31,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:33,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:35,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:35,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:35,392 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:35,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:35,398 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:35,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:37,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:39,140 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:00:40,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:41,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:41,738 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:41,738 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:41,744 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:41,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:43,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:43,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:43,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:43,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:45,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:46,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:46,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:46,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:46,376 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:46,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:47,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:51,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:54,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:54,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:54,926 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:54,932 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:55,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:56,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:57,780 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:58,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:58,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:58,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:00,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:00,257 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:00,258 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:00,263 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:00,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:01,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:02,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:05,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:05,855 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:05,855 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:05,861 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:06,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:07,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:07,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:09,142 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:01:10,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:10,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:10,307 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:10,313 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:10,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:11,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:13,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:13,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:13,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:13,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:15,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:15,646 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:15,646 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:15,652 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:15,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:17,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:18,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:21,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:21,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:21,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:21,244 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:21,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:23,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:24,244 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:28,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:28,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:28,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:29,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:29,128 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:29,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:29,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:29,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:30,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:31,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:34,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:34,456 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:34,456 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:34,462 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:34,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:35,462 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:37,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:38,934 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:38,935 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:38,935 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:38,941 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:39,144 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:01:39,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:41,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:41,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:43,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:43,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:43,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:45,247 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:45,247 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:45,248 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:45,254 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:45,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:46,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:47,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:50,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:50,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:50,450 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:50,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:50,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:51,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:53,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:56,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:58,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:58,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:58,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:58,843 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:58,844 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:58,844 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:58,851 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:59,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:01,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:01,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:05,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:05,572 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:05,572 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:05,578 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:05,850 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:07,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:07,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:09,147 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:02:11,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:11,182 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:11,182 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:11,188 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:11,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:13,188 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:13,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:13,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:13,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:13,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:16,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:16,485 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:16,485 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:16,491 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:16,860 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:18,526 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:19,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:20,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:20,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:20,650 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:20,657 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:20,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:23,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:23,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:25,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:25,471 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:25,471 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:25,477 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:25,872 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:27,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:28,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:28,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:28,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:29,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:31,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:31,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:31,767 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:31,774 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:31,879 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:33,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:34,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:35,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:35,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:35,926 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:35,932 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:36,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:37,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:39,152 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:02:39,754 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:39,755 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:39,755 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:39,761 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:39,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:40,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:41,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:43,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:43,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:43,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:45,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:45,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:45,033 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:45,040 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:45,896 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:46,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:47,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:51,074 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:51,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:51,580 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:51,580 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:51,586 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:51,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:53,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:56,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:57,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:57,191 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:57,192 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:57,197 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:57,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:58,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:58,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:58,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:59,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:01,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:02,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:02,975 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:02,975 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:02,982 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:03,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:05,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:07,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:08,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:08,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:08,744 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:08,750 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:08,924 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:09,151 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:03:11,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:12,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:13,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:13,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:13,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:13,787 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:13,788 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:13,788 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:13,794 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:13,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:15,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:17,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:18,908 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:18,909 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:18,909 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:18,915 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:18,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:21,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:22,916 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:23,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:23,705 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:23,705 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:23,711 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:23,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:25,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:28,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:28,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:28,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:28,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:28,692 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:28,693 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:28,693 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:28,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:28,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:31,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:33,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:33,234 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:33,235 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:33,241 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:33,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:34,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:35,962 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:39,156 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:03:40,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:40,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:40,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:40,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:40,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:40,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:41,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:43,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:43,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:43,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:45,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:45,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:45,995 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:45,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:46,001 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:46,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:47,976 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:50,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:50,507 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:50,508 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:50,514 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:50,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:51,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:53,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:54,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:54,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:54,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:54,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:54,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:56,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:57,988 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:58,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:58,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:58,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:59,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:59,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:59,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:59,320 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:59,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:01,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:02,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:03,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:03,838 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:03,838 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:03,844 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:03,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:06,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:07,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:08,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:08,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:08,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:08,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:09,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:09,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:04:10,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:12,091 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:12,092 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:12,092 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:12,098 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:13,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:13,099 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:13,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:13,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:14,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:16,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:16,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:16,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:16,217 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:17,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:18,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:18,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:21,523 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:21,524 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:21,524 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:21,530 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:22,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:23,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:24,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:27,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:27,862 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:27,863 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:27,868 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:28,025 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:28,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:28,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:28,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:28,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:30,029 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:33,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:33,253 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:33,253 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:33,259 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:34,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:34,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:36,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:38,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:38,421 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:38,421 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:38,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:39,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:39,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:04:40,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:40,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:42,699 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:42,700 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:42,700 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:42,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:43,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:43,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:43,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:43,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:45,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:46,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:47,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:47,207 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:47,207 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:47,213 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:48,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:50,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:51,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:51,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:51,268 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:51,268 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:51,274 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:52,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:54,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:56,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:56,132 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:56,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:56,138 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:57,057 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:57,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:58,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:58,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:58,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:58,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:01,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:01,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:01,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:01,137 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:02,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:02,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:04,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:06,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:06,120 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:06,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:06,127 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:07,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:08,073 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:08,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:09,159 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:05:11,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:11,460 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:11,460 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:11,466 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:12,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:13,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:13,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:13,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:13,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:14,081 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:15,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:15,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:15,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:15,870 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:16,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:18,086 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:18,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:20,930 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:20,931 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:20,931 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:20,937 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:21,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:23,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:23,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:26,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:26,291 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:26,291 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:26,302 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:27,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:27,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:28,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:28,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:28,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:29,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:31,656 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:31,656 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:31,657 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:31,662 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:32,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:33,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:34,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:35,986 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:35,987 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:35,987 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:35,993 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:36,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:37,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:39,162 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:05:40,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:42,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:42,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:42,184 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:42,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:43,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:43,120 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:43,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:43,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:43,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:45,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:46,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:46,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:46,476 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:46,482 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:47,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:47,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:51,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:51,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:51,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:51,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:51,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:52,129 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:53,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:56,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:56,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:56,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:56,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:56,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:57,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:57,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:58,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:58,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:58,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:00,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:00,355 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:00,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:00,361 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:01,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:01,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:02,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:05,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:05,111 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:05,111 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:05,117 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:05,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:07,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:08,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:09,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:06:12,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:12,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:12,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:12,349 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:13,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:13,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:13,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:13,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:13,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:17,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:17,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:17,044 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:17,050 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:17,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:19,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:19,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:21,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:21,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:21,703 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:21,708 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:22,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:23,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:24,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:26,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:26,366 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:26,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:26,373 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:27,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:27,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:28,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:28,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:28,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:30,622 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:30,623 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:30,623 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:30,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:30,629 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:31,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:31,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:35,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:35,421 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:35,421 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:35,427 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:36,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:36,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:37,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:39,166 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:06:39,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:39,997 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:39,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:40,003 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:40,195 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:41,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:42,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:43,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:43,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:43,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:47,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:48,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:48,393 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:48,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:48,400 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:49,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:49,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:53,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:55,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:55,795 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:55,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:55,801 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:56,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:57,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:58,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:58,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:58,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:58,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:01,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:01,364 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:01,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:01,371 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:02,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:03,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:04,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:05,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:05,822 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:05,822 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:05,828 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:06,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:07,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:09,171 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:07:10,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:11,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:11,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:11,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:11,740 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:12,231 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:13,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:13,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:13,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:15,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:16,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:16,224 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:16,225 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:16,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:16,237 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:17,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:20,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:20,810 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:20,810 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:20,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:20,816 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:21,246 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:23,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:25,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:25,347 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:25,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:25,353 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:26,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:26,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:27,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:28,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:28,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:28,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:30,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:30,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:30,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:30,780 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:31,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:31,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:33,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:35,712 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:35,713 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:35,713 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:35,719 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:36,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:37,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:37,719 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:39,171 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:07:40,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:40,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:40,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:40,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:41,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:42,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:43,274 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:43,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:43,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:43,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:45,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:45,585 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:45,585 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:45,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:46,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:47,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:48,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:50,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:50,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:50,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:50,293 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:51,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:51,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:54,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:54,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:54,376 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:54,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:54,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:55,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:55,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:58,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:58,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:58,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:59,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:00,668 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:00,668 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:00,669 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:00,675 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:01,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:01,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:04,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:05,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:05,134 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:05,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:05,139 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:05,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:07,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:09,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:08:10,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:10,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:10,585 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:10,585 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:10,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:11,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:11,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:13,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:15,595 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:15,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:15,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:15,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:15,846 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:16,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:17,338 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:20,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:21,056 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:21,057 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:21,057 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:21,063 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:21,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:23,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:26,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:28,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:28,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:28,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:29,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:29,199 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:29,199 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:29,205 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:29,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:31,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:31,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:33,523 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:33,524 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:33,524 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:33,530 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:34,372 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:35,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:36,531 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:38,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:38,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:38,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:38,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:38,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:39,175 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:08:39,382 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:42,176 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:43,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:43,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:43,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:44,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:44,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:44,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:44,893 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:45,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:47,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:47,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:52,930 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:53,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:53,559 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:53,559 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:53,565 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:54,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:55,415 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:58,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:58,142 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:58,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:58,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:58,147 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:58,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:58,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:58,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:58,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:59,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:03,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:04,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:04,334 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:04,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:04,343 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:04,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:05,427 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:09,177 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:09:09,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:10,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:10,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:10,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:10,100 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:10,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:11,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:13,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:14,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:14,844 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:14,845 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:14,845 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:14,850 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:15,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:17,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:19,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:19,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:19,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:19,894 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:19,900 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:20,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:21,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:24,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:25,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:25,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:25,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:25,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:26,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:27,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:28,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:28,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:28,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:30,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:30,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:30,655 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:30,655 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:30,661 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:31,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:31,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:35,054 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:35,055 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:35,056 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:35,061 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:35,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:36,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:37,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:39,179 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:09:41,180 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:43,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:43,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:43,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:44,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:44,346 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:44,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:44,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:44,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:45,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:46,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:48,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:48,476 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:48,476 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:48,482 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:48,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:49,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:51,483 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:53,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:53,367 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:53,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:53,373 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:53,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:55,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:57,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:57,519 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:57,520 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:57,520 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:57,526 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:58,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:58,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:58,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:58,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:59,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:02,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:05,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:05,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:05,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:05,016 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:05,503 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:07,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:08,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:09,181 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:10:10,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:10,484 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:10,485 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:10,490 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:10,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:11,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:13,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:13,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:13,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:15,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:15,239 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:15,239 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:15,245 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:15,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:17,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:19,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:19,795 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:19,796 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:19,796 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:19,802 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:20,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:21,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:24,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:25,952 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:25,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:25,953 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:25,958 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:26,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:27,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:28,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:28,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:28,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:30,622 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:31,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:31,769 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:31,770 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:31,775 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:32,536 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:33,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:35,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:35,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:35,962 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:35,962 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:35,968 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:36,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:37,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:39,187 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:10:41,064 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:41,064 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:41,065 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:41,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:41,070 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:41,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:43,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:43,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:43,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:43,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:46,068 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:46,069 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:46,069 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:46,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:46,075 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:46,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:47,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:51,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:51,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:51,358 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:51,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:51,364 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:51,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:53,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:56,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:56,302 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:56,302 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:56,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:56,307 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:56,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:57,567 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:58,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:58,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:58,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:00,795 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:00,795 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:00,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:00,801 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:01,572 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:01,572 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:01,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:06,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:07,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:07,123 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:07,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:07,129 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:07,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:09,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:11:09,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:12,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:13,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:13,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:13,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:15,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:15,665 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:15,665 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:15,670 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:16,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:17,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:17,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:20,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:20,411 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:20,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:20,427 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:20,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:21,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:23,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:24,876 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:24,877 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:24,877 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:24,883 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:25,600 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:25,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:28,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:28,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:28,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:28,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:29,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:29,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:29,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:29,328 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:29,603 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:31,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:33,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:33,900 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:33,900 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:33,900 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:33,906 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:34,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:35,612 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:38,906 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:39,191 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:11:42,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:42,445 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:42,445 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:42,451 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:42,617 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:43,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:43,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:43,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:43,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:44,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:47,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:47,187 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:47,187 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:47,192 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:47,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:49,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:50,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:53,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:53,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:53,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:53,178 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:53,630 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:55,633 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:56,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:57,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:57,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:57,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:57,734 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:58,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:58,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:58,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:58,636 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:59,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:01,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:04,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:04,330 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:04,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:04,336 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:04,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:05,645 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:07,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:09,190 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:12:09,931 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:09,932 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:09,932 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:09,938 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:10,648 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:11,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:12,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:13,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:13,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:13,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:16,062 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:16,063 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:16,063 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:16,069 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:16,655 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:17,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:18,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:20,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:20,785 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:20,785 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:20,791 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:21,666 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:21,666 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:23,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:25,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:25,173 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:25,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:25,179 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:25,669 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:27,672 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:28,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:28,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:28,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:29,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:29,542 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:29,542 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:29,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:29,547 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:29,674 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:31,677 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:34,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:34,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:34,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:34,027 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:34,680 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:35,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:35,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:38,690 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:38,691 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:38,691 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:38,697 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:39,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:12:39,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:39,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:40,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:43,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:43,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:43,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:43,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:43,605 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:43,605 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:43,611 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:43,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:45,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:45,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:47,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:47,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:47,619 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:47,625 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:47,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:49,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:50,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:54,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:54,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:54,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:54,016 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:54,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:55,707 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:56,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:58,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:58,315 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:58,315 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:58,320 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:58,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:58,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:58,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:58,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:59,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:01,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:04,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:04,531 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:04,531 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:04,536 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:04,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:05,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:07,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:09,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:13:09,719 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:09,719 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:09,720 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:09,725 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:09,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:11,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:12,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:13,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:13,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:14,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:14,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:14,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:14,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:14,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:15,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:18,225 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:23,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:23,152 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:23,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:23,158 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:23,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:24,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:25,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:27,902 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:27,902 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:27,903 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:27,908 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:28,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:28,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:28,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:28,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:29,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:29,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:34,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:36,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:36,241 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:36,241 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:36,247 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:36,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:37,759 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:39,197 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:13:40,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:41,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:41,129 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:41,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:41,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:41,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:41,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:43,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:43,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:43,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:45,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:46,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:46,655 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:46,655 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:46,662 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:46,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:47,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:50,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:52,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:52,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:52,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:52,217 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:52,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:53,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:56,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:56,291 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:56,292 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:56,292 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:56,298 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:56,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:57,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:58,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:58,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:58,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:00,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:00,933 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:00,934 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:00,940 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:01,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:01,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:01,980 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:05,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:05,390 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:05,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:05,396 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:05,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:07,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:07,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:09,200 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:14:10,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:10,310 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:10,310 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:10,316 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:10,796 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:11,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:13,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:13,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:13,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:14,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:14,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:14,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:14,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:15,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:15,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:18,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:20,232 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:20,233 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:20,233 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:20,238 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:20,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:21,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:24,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:28,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:28,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:28,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:29,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:29,746 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:29,747 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:29,747 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:29,753 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:29,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:31,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:34,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:35,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:35,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:35,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:35,163 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:35,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:35,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:39,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:14:40,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:41,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:41,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:41,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:41,860 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:42,832 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:43,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:43,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:43,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:43,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:45,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:47,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:47,228 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:47,228 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:47,234 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:47,838 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:49,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:51,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:55,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:55,631 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:55,631 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:55,637 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:55,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:56,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:57,850 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:58,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:58,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:58,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:00,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:00,234 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:00,235 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:00,241 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:00,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:01,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:02,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:06,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:06,369 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:06,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:06,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:06,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:07,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:07,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:09,207 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:15:11,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:11,952 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:11,952 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:11,958 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:12,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:12,958 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:13,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:13,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:13,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:16,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:16,467 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:16,467 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:16,473 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:16,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:17,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:18,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:22,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:22,952 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:22,952 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:22,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:23,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:23,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:23,959 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:28,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:28,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:28,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:28,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:28,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:28,643 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:28,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:28,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:29,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:29,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:33,699 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:33,700 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:33,700 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:33,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:33,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:34,707 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:35,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:39,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:39,050 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:39,051 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:39,056 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:39,206 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:15:39,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:39,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:40,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:43,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:43,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:43,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:43,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:43,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:43,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:43,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:43,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:45,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:45,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:49,246 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:49,247 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:49,247 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:49,253 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:49,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:49,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:51,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:55,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:55,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:55,474 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:55,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:55,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:56,480 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:57,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:58,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:58,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:58,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:59,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:59,793 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:59,793 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:59,799 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:59,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:01,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:01,925 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:04,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:04,304 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:04,304 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:04,309 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:04,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:05,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:07,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:08,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:08,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:08,650 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:08,656 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:08,932 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:09,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:16:09,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:13,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:13,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:13,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:14,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:14,102 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:14,102 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:14,108 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:14,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:15,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:19,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:20,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:20,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:20,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:20,814 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:20,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:21,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:24,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:28,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:28,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:28,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:29,246 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:29,247 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:29,247 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:29,254 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:29,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:29,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:30,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:33,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:33,781 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:33,781 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:33,786 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:33,961 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:35,787 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:35,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:38,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:38,946 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:38,946 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:38,952 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:38,967 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:39,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:16:39,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:41,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:43,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:43,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:43,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:43,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:43,699 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:43,699 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:43,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:43,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:45,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:46,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:48,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:48,110 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:48,110 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:48,116 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:48,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:49,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:52,116 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:52,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:52,145 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:52,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:52,151 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:52,984 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:53,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:56,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:56,694 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:56,694 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:56,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:56,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:57,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:57,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:58,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:58,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:58,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:01,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:01,380 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:01,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:01,389 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:01,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:03,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:04,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:06,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:06,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:06,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:06,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:07,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:08,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:08,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:09,213 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:17:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:13,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:13,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:13,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:15,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:15,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:15,044 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:15,050 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:16,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:16,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:19,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:19,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:19,811 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:19,811 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:19,818 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:20,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:22,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:23,918 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:23,919 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:23,919 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:23,925 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:24,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:24,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:26,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:28,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:28,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:28,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:30,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:30,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:30,629 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:30,630 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:30,635 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:31,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:32,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:35,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:38,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:38,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:38,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:38,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:39,040 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:39,215 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:17:40,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:41,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:43,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:43,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:43,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:44,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:45,000 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:45,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:45,006 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:45,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:46,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:47,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:49,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:49,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:49,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:49,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:50,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:50,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:52,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:54,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:54,121 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:54,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:54,126 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:55,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:56,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:58,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:58,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:58,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:58,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:58,882 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:58,882 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:58,888 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:59,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:00,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:03,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:03,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:03,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:03,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:03,243 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:04,073 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:04,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:08,244 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:08,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:08,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:08,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:08,292 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:09,077 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:09,218 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:18:10,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:13,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:13,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:16,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:16,485 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:16,485 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:16,490 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:17,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:18,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:19,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:22,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:22,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:22,044 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:22,050 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:22,092 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:24,096 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:25,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:26,735 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:26,736 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:26,737 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:26,742 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:27,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:28,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:28,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:28,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:28,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:30,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:31,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:31,214 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:31,214 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:31,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:32,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:32,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:36,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:38,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:38,197 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:38,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:38,204 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:39,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:39,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:18:40,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:42,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:43,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:43,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:43,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:43,561 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:43,561 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:43,567 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:44,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:46,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:47,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:48,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:48,062 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:48,062 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:48,068 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:48,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:50,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:52,902 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:52,902 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:52,903 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:52,903 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:52,908 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:53,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:54,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:57,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:57,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:57,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:57,893 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:58,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:58,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:58,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:58,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:58,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:00,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:03,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:03,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:03,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:03,898 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:03,904 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:04,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:06,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:08,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:09,222 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:19:10,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:10,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:10,013 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:10,019 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:10,147 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:12,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:13,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:13,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:14,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:14,522 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:14,522 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:14,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:14,528 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:15,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:16,155 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:19,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:19,133 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:19,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:19,139 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:19,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:20,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:20,162 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:24,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:24,218 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:24,218 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:24,224 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:25,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:25,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:26,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:28,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:28,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:28,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:28,782 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:28,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:28,783 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:28,789 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:29,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:30,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:30,825 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:34,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:34,261 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:34,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:34,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:35,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:36,182 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:36,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:39,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:39,182 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:39,182 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:39,188 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:39,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:19:40,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:40,187 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:42,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:43,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:43,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:44,847 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:44,847 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:44,847 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:44,853 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:45,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:46,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:47,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:49,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:49,470 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:49,470 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:49,476 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:50,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:50,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:53,476 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:55,692 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:55,693 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:55,693 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:55,699 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:56,202 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:58,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:58,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:58,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:58,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:58,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:59,924 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:59,925 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:59,925 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:59,930 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:00,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:02,211 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:03,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:04,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:04,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:04,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:04,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:05,213 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:06,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:09,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:20:09,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:12,511 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:12,512 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:12,512 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:12,522 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:13,220 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:13,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:13,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:13,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:14,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:14,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:19,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:19,215 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:19,215 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:19,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:19,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:20,231 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:20,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:23,908 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:23,909 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:23,909 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:23,914 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:24,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:25,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:26,238 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:28,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:28,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:28,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:29,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:29,169 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:29,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:29,175 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:29,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:30,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:31,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:35,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:35,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:35,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:35,155 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:35,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:36,250 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:37,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:39,229 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:20:40,799 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:40,800 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:40,800 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:40,806 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:41,254 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:42,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:42,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:43,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:43,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:45,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:45,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:45,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:45,440 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:46,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:46,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:48,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:51,204 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:51,205 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:51,205 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:51,211 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:51,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:52,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:54,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:56,240 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:56,241 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:56,241 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:56,247 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:56,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:58,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:58,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:58,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:58,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:59,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:00,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:00,616 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:00,616 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:00,621 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:01,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:02,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:04,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:05,915 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:05,916 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:05,916 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:05,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:06,283 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:08,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:09,231 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:21:09,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:09,922 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:09,922 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:09,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:09,928 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:10,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:12,292 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:13,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:13,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:13,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:14,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:14,572 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:14,573 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:14,578 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:15,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:15,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:16,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:19,538 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:19,539 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:19,539 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:19,545 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:20,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:20,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:21,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:23,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:23,865 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:23,866 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:23,871 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:24,308 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:26,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:26,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:28,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:28,229 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:28,229 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:28,235 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:28,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:28,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:28,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:28,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:30,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:32,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:35,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:35,662 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:35,662 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:35,668 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:36,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:37,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:38,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:39,233 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:21:40,246 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:40,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:40,246 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:40,253 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:40,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:42,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:43,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:43,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:43,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:43,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:45,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:45,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:45,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:45,411 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:46,336 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:46,336 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:48,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:49,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:49,815 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:49,815 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:49,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:50,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:52,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:53,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:55,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:55,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:55,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:55,202 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:55,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:56,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:58,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:58,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:58,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:59,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:03,835 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:03,836 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:03,836 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:03,842 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:04,355 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:04,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:06,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:07,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:07,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:07,866 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:07,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:08,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:09,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:22:10,240 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:10,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:13,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:13,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:13,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:14,479 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:14,480 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:14,480 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:14,486 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:15,368 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:15,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:16,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:20,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:20,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:20,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:20,619 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:20,625 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:21,376 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:22,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:25,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:28,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:28,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:28,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:28,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:28,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:28,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:28,642 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:29,385 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:30,388 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:30,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:32,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:32,437 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:32,437 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:32,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:33,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:34,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:36,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:36,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:36,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:36,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:36,716 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:37,395 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:38,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:39,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:22:41,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:41,003 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:41,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:41,009 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:41,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:42,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:42,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:43,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:43,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:45,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:45,455 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:45,456 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:45,461 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:46,409 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:46,409 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:47,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:50,751 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:50,752 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:50,752 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:50,758 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:51,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:52,415 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:52,758 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:57,758 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:58,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:58,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:58,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:58,868 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:58,868 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:58,868 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:58,875 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:59,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:00,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:02,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:03,927 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:03,927 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:03,927 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:03,933 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:04,427 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:06,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:07,934 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:08,765 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:08,766 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:08,766 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:08,772 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:09,243 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:23:09,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:10,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:13,244 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:13,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:13,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:13,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:13,775 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:13,776 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:13,776 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:13,781 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:14,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:16,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:18,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:18,271 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:18,271 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:18,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:18,276 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:18,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:20,449 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:23,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:24,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:24,336 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:24,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:24,342 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:24,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:26,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:28,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:28,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:28,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:28,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:28,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:28,671 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:28,672 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:28,677 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:29,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:30,462 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:33,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:33,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:33,994 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:33,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:34,000 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:34,465 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:36,469 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:39,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:39,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:23:39,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:39,793 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:39,794 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:39,799 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:40,474 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:40,474 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:43,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:44,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:47,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:47,684 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:47,685 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:47,690 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:48,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:48,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:49,725 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:51,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:51,948 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:51,949 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:51,955 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:52,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:54,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:54,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:56,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:56,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:56,820 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:56,826 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:57,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:58,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:58,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:58,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:58,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:00,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:03,950 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:03,951 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:03,951 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:03,958 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:04,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:05,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:06,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:09,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:09,091 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:09,091 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:09,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:09,249 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:24:09,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:10,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:11,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:13,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:13,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:13,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:14,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:14,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:14,459 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:14,465 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:14,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:16,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:16,516 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:18,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:18,970 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:18,970 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:18,976 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:19,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:20,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:21,976 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:26,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:27,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:27,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:27,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:27,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:27,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:28,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:28,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:28,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:28,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:32,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:32,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:32,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:32,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:32,446 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:32,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:34,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:36,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:36,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:36,883 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:36,888 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:37,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:37,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:38,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:39,248 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:24:42,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:42,445 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:42,445 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:42,451 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:42,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:43,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:43,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:43,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:44,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:47,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:47,190 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:47,190 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:47,196 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:47,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:48,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:49,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:51,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:51,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:51,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:51,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:51,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:52,560 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:54,480 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:55,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:55,911 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:55,911 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:55,916 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:56,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:58,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:58,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:58,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:58,567 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:59,606 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:01,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:01,402 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:01,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:01,408 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:01,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:02,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:05,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:06,119 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:06,120 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:06,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:06,126 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:06,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:08,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:09,250 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:25:10,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:10,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:10,872 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:10,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:10,877 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:11,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:12,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:13,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:13,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:13,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:16,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:16,249 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:16,250 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:16,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:16,255 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:16,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:18,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:20,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:20,345 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:20,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:20,351 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:20,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:21,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:22,597 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:26,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:26,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:26,853 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:26,853 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:26,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:27,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:28,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:28,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:28,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:28,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:31,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:34,947 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:34,948 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:34,948 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:34,954 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:35,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:36,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:36,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:39,253 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:25:40,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:40,182 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:40,182 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:40,187 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:40,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:42,188 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:42,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:43,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:43,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:45,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:45,350 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:45,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:45,355 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:45,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:46,624 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:47,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:49,868 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:49,869 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:49,869 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:49,875 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:50,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:50,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:52,876 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:56,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:56,549 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:56,549 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:56,555 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:56,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:58,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:58,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:58,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:58,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:58,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:01,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:01,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:01,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:01,734 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:02,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:02,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:03,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:07,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:07,680 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:07,680 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:07,686 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:08,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:08,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:09,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:26:09,256 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:12,944 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:12,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:12,945 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:12,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:13,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:13,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:13,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:13,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:14,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:14,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:19,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:19,326 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:19,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:19,332 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:19,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:20,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:20,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:25,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:26,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:26,091 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:26,091 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:26,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:26,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:28,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:28,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:28,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:28,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:30,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:31,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:31,438 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:31,438 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:31,444 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:31,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:32,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:35,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:35,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:35,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:35,864 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:35,870 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:36,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:36,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:39,260 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:26:41,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:41,200 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:41,200 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:41,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:41,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:41,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:42,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:43,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:43,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:43,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:46,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:46,585 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:46,585 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:46,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:46,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:46,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:48,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:50,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:50,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:50,894 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:50,900 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:51,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:51,900 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:52,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:55,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:55,714 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:55,714 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:55,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:56,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:56,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:57,720 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:58,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:58,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:58,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:02,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:02,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:02,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:02,060 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:02,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:03,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:04,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:06,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:06,526 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:06,526 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:06,532 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:06,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:08,532 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:08,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:09,259 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:27:11,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:11,327 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:11,327 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:11,333 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:11,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:12,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:13,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:13,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:13,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:13,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:15,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:15,810 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:15,810 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:15,816 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:16,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:16,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:18,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:20,070 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:20,070 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:20,070 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:20,076 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:20,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:22,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:24,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:28,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:28,124 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:28,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:28,129 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:28,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:28,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:28,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:28,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:29,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:30,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:33,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:33,401 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:33,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:33,407 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:33,754 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:34,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:35,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:39,264 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:27:41,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:41,718 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:41,719 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:41,719 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:41,725 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:41,763 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:42,766 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:43,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:43,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:43,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:46,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:47,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:47,004 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:47,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:47,009 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:47,769 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:48,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:51,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:51,680 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:51,680 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:51,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:51,687 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:51,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:52,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:56,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:56,052 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:56,052 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:56,058 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:56,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:56,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:57,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:58,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:58,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:58,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:01,071 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:01,072 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:01,072 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:01,078 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:01,786 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:02,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:02,787 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:07,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:07,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:07,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:07,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:07,173 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:07,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:08,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:09,263 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:28:11,848 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:11,849 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:11,849 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:11,854 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:12,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:12,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:12,855 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:13,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:13,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:16,491 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:16,491 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:16,491 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:16,497 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:16,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:18,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:18,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:20,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:20,723 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:20,723 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:20,728 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:20,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:22,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:23,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:28,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:28,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:28,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:29,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:29,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:29,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:29,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:29,190 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:29,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:30,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:34,225 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:34,283 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:34,284 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:34,284 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:34,290 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:34,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:36,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:38,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:38,604 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:38,604 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:38,610 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:38,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:39,266 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:28:39,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:40,834 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:43,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:43,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:43,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:43,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:43,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:43,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:43,892 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:44,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:44,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:44,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:49,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:49,561 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:49,561 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:49,567 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:49,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:50,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:50,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:54,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:54,369 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:54,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:54,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:54,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:56,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:56,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:58,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:58,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:58,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:58,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:58,690 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:58,690 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:58,696 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:58,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:00,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:01,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:03,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:03,046 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:03,046 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:03,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:03,860 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:04,863 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:07,052 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:08,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:08,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:08,654 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:08,660 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:08,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:09,268 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:29:10,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:12,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:12,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:12,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:12,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:12,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:13,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:13,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:13,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:14,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:17,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:18,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:18,078 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:18,078 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:18,083 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:18,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:18,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:23,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:26,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:26,203 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:26,203 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:26,208 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:26,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:26,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:28,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:28,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:28,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:28,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:30,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:30,905 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:30,905 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:30,912 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:31,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:32,896 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:33,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:36,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:36,028 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:36,029 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:36,034 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:36,902 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:36,902 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:39,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:39,270 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:29:42,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:42,968 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:42,968 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:42,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:43,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:43,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:43,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:43,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:44,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:44,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:47,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:47,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:47,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:47,520 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:47,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:48,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:50,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:55,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:56,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:56,018 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:56,018 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:56,025 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:56,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:56,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:58,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:58,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:58,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:00,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:00,723 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:00,724 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:00,724 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:00,730 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:00,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:02,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:05,765 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:06,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:06,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:06,307 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:06,313 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:06,932 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:08,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:09,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:30:11,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:11,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:11,196 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:11,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:11,202 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:11,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:12,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:13,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:13,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:13,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:16,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:16,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:16,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:16,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:16,350 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:16,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:18,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:21,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:22,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:22,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:22,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:22,400 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:22,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:24,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:26,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:26,916 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:26,917 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:26,917 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:26,923 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:26,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:28,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:28,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:28,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:28,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:31,204 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:31,205 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:31,205 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:31,211 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:31,962 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:32,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:32,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:36,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:36,042 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:36,042 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:36,048 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:36,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:36,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:38,048 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:39,274 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:30:41,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:41,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:41,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:41,322 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:41,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:42,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:43,323 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:43,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:43,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:43,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:47,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:47,251 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:47,252 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:47,258 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:47,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:48,984 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:49,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:52,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:52,001 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:52,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:52,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:52,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:52,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:55,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:57,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:57,627 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:57,627 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:57,634 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:57,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:58,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:58,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:58,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:58,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:00,758 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:02,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:02,324 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:02,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:02,330 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:03,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:05,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:06,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:09,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:31:10,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:10,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:10,458 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:10,464 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:11,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:11,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:13,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:13,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:13,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:14,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:14,955 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:14,955 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:14,961 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:15,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:16,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:17,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:19,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:19,491 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:19,491 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:19,497 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:20,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:21,024 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:22,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:24,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:24,781 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:24,781 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:24,787 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:25,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:27,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:27,787 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:28,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:28,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:28,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:29,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:29,029 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:29,029 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:29,034 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:29,035 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:31,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:33,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:38,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:38,021 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:38,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:38,027 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:38,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:39,027 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:39,047 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:39,279 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:31:43,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:43,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:43,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:43,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:43,841 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:43,842 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:43,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:44,051 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:44,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:45,053 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:48,099 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:48,100 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:48,100 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:48,106 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:49,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:49,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:50,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:52,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:52,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:52,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:52,815 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:53,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:55,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:55,815 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:57,067 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:57,068 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:57,068 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:57,074 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:58,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:58,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:58,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:58,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:59,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:01,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:03,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:03,373 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:03,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:03,379 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:04,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:05,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:07,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:09,284 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:32:09,931 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:09,932 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:09,932 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:09,938 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:10,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:11,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:12,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:13,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:14,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:14,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:14,608 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:14,614 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:15,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:17,092 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:18,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:19,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:19,982 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:19,983 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:19,989 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:20,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:21,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:23,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:25,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:25,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:25,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:25,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:26,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:27,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:28,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:28,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:28,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:29,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:30,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:30,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:30,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:30,734 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:31,109 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:33,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:34,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:37,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:37,142 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:37,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:37,148 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:38,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:39,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:39,283 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:32:40,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:43,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:43,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:43,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:43,524 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:43,626 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:43,626 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:43,632 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:44,123 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:45,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:45,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:50,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:50,526 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:50,526 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:50,532 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:51,131 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:51,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:53,135 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:54,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:54,675 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:54,675 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:54,681 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:55,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:56,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:57,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:58,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:58,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:58,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:59,052 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:59,052 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:59,053 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:59,059 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:59,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:01,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:02,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:03,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:03,471 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:03,471 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:03,477 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:04,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:05,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:07,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:08,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:08,141 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:08,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:08,147 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:08,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:09,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:09,285 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:33:12,622 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:12,623 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:12,624 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:12,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:12,630 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:13,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:13,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:13,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:15,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:18,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:18,162 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:18,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:18,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:18,168 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:18,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:19,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:23,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:23,021 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:23,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:23,027 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:23,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:24,027 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:25,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:27,479 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:27,480 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:27,480 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:27,487 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:28,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:28,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:28,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:28,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:29,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:29,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:32,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:32,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:32,498 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:32,504 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:33,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:35,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:35,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:37,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:37,363 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:37,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:37,369 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:38,195 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:39,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:39,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:33:41,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:41,482 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:41,483 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:41,483 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:41,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:42,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:43,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:43,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:43,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:43,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:46,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:46,014 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:46,014 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:46,020 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:46,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:47,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:47,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:51,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:51,396 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:51,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:51,416 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:52,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:52,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:53,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:57,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:58,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:58,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:58,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:59,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:59,731 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:59,731 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:59,738 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:00,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:01,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:02,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:03,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:03,833 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:03,833 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:03,839 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:04,226 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:05,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:07,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:08,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:08,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:08,279 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:08,285 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:09,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:09,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:09,290 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:34:13,291 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:13,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:13,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:13,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:13,339 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:13,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:13,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:14,238 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:15,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:18,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:18,188 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:18,188 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:18,194 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:18,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:19,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:19,246 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:23,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:23,376 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:23,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:23,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:24,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:24,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:25,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:28,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:28,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:28,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:29,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:29,274 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:29,274 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:29,280 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:30,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:30,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:31,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:34,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:34,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:34,637 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:34,642 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:35,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:35,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:37,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:39,295 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:34:41,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:41,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:41,616 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:41,617 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:41,623 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:42,272 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:43,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:43,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:43,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:43,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:45,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:45,942 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:45,942 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:45,948 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:46,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:46,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:47,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:51,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:51,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:51,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:51,017 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:51,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:52,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:53,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:55,503 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:55,504 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:55,505 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:55,515 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:56,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:57,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:57,516 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:58,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:58,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:58,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:00,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:00,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:00,202 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:00,211 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:00,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:01,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:03,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:05,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:05,282 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:05,283 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:05,288 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:05,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:07,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:08,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:09,294 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:35:09,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:09,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:09,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:09,740 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:10,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:11,308 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:13,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:13,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:13,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:13,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:14,112 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:14,113 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:14,113 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:14,118 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:14,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:15,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:19,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:21,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:21,259 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:21,259 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:21,265 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:21,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:23,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:24,266 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:26,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:26,206 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:26,206 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:26,211 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:26,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:27,328 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:28,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:28,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:28,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:29,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:31,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:31,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:31,286 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:31,292 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:31,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:33,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:35,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:35,761 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:35,762 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:35,762 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:35,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:36,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:37,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:39,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:35:41,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:42,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:42,095 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:42,095 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:42,101 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:42,344 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:43,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:43,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:43,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:43,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:46,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:48,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:48,006 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:48,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:48,012 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:48,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:49,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:52,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:52,718 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:52,719 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:52,719 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:52,724 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:53,357 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:55,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:57,725 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:58,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:58,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:58,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:00,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:00,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:00,969 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:00,975 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:01,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:03,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:03,369 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:05,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:05,616 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:05,616 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:05,622 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:06,372 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:07,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:08,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:09,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:36:10,067 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:10,068 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:10,069 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:10,074 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:10,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:11,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:13,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:13,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:13,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:13,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:17,850 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:17,851 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:17,851 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:17,857 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:18,386 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:18,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:19,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:23,736 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:23,737 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:23,737 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:23,743 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:24,395 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:24,743 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:25,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:28,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:28,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:28,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:29,844 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:29,845 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:29,845 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:29,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:29,851 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:30,402 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:31,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:34,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:34,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:34,971 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:34,971 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:34,977 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:35,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:37,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:39,301 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:36:39,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:39,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:39,439 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:39,444 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:40,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:40,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:41,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:43,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:43,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:43,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:43,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:43,889 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:43,890 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:43,895 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:44,420 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:45,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:45,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:49,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:49,403 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:49,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:49,409 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:49,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:51,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:51,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:55,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:55,363 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:55,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:55,370 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:55,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:57,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:57,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:58,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:58,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:58,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:00,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:00,268 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:00,268 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:00,274 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:00,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:01,442 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:03,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:05,118 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:05,119 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:05,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:05,126 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:05,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:07,449 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:09,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:09,303 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:37:09,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:09,765 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:09,765 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:09,771 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:10,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:11,454 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:13,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:13,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:13,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:14,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:14,493 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:14,493 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:14,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:14,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:15,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:15,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:18,924 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:18,925 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:18,925 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:18,931 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:19,462 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:19,932 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:21,469 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:23,563 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:23,564 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:23,565 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:23,570 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:24,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:25,474 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:25,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:28,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:28,161 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:28,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:28,166 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:28,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:28,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:28,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:28,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:29,480 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:30,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:33,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:33,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:33,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:33,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:33,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:35,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:36,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:37,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:37,251 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:37,251 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:37,257 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:37,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:39,305 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:37:39,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:41,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:41,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:41,741 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:41,742 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:41,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:42,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:43,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:43,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:43,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:43,501 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:46,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:47,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:47,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:47,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:47,017 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:47,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:49,508 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:52,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:52,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:52,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:52,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:52,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:52,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:53,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:56,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:56,148 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:56,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:56,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:56,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:57,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:57,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:58,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:58,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:58,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:01,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:01,509 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:01,510 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:01,516 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:01,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:02,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:03,527 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:06,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:06,161 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:06,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:06,167 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:06,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:07,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:08,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:09,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:38:13,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:13,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:13,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:13,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:13,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:13,565 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:13,565 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:13,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:14,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:15,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:17,799 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:17,799 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:17,800 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:17,806 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:18,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:18,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:19,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:22,263 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:22,264 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:22,265 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:22,270 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:22,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:23,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:24,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:27,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:27,424 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:27,424 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:27,431 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:27,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:28,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:28,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:28,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:29,560 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:29,574 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:32,515 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:32,516 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:32,517 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:32,523 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:32,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:33,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:35,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:37,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:37,991 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:37,991 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:37,997 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:38,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:39,309 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:38:39,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:41,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:42,519 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:42,520 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:42,520 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:42,526 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:42,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:43,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:43,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:43,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:43,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:46,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:46,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:46,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:46,779 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:46,784 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:47,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:47,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:51,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:51,147 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:51,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:51,153 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:51,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:52,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:53,589 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:55,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:55,681 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:55,682 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:55,688 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:56,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:57,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:57,688 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:58,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:58,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:58,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:00,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:00,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:00,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:00,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:00,597 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:01,600 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:03,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:08,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:09,315 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:39:13,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:13,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:13,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:13,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:13,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:13,588 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:13,589 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:13,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:13,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:15,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:18,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:18,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:18,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:18,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:18,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:19,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:19,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:22,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:22,667 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:22,667 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:22,673 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:23,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:23,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:24,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:27,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:27,369 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:27,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:27,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:27,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:28,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:28,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:28,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:29,630 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:30,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:31,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:31,785 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:31,786 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:31,792 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:32,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:33,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:35,830 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:36,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:36,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:36,458 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:36,464 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:36,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:37,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:39,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:39:40,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:40,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:40,925 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:40,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:40,932 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:41,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:43,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:43,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:43,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:43,648 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:45,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:45,559 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:45,559 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:45,565 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:45,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:46,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:47,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:50,864 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:50,865 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:50,866 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:50,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:51,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:51,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:51,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:55,840 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:55,841 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:55,842 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:55,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:56,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:57,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:57,848 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:58,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:58,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:58,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:01,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:01,170 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:01,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:01,176 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:01,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:03,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:03,672 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:08,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:09,318 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:40:09,550 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:09,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:09,551 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:09,556 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:09,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:11,680 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:13,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:13,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:13,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:13,676 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:14,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:14,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:14,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:14,701 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:15,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:15,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:18,738 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:20,550 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:20,552 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:20,552 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:20,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:20,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:21,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:24,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:25,108 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:25,108 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:25,109 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:25,114 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:25,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:27,701 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:28,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:28,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:28,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:29,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:33,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:33,313 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:33,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:33,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:33,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:35,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:35,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:38,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:38,019 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:38,019 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:38,025 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:38,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:39,318 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:40:39,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:41,319 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:43,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:43,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:43,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:45,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:45,281 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:45,281 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:45,287 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:45,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:46,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:47,722 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:49,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:49,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:49,807 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:49,813 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:50,724 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:51,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:51,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:56,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:58,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:58,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:58,203 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:58,209 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:58,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:58,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:58,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:58,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:59,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:02,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:03,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:03,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:03,448 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:03,454 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:03,738 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:05,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:08,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:08,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:08,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:08,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:08,156 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:08,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:09,320 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:41:09,747 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:13,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:13,279 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:13,279 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:13,279 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:13,285 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:13,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:13,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:13,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:15,754 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:18,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:18,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:18,977 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:18,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:18,984 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:19,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:21,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:24,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:24,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:24,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:24,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:24,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:24,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:25,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:28,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:28,271 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:28,271 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:28,276 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:28,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:28,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:28,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:28,771 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:29,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:29,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:33,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:33,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:33,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:33,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:33,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:35,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:35,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:39,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:41:39,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:39,526 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:39,526 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:39,532 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:39,784 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:40,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:41,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:43,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:43,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:43,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:45,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:46,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:46,271 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:46,271 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:46,277 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:46,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:47,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:51,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:53,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:53,772 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:53,772 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:53,778 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:53,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:55,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:56,779 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:58,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:58,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:58,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:00,987 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:00,987 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:00,987 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:00,993 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:01,808 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:01,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:02,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:05,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:05,463 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:05,464 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:05,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:05,811 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:07,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:07,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:09,324 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:42:09,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:09,941 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:09,942 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:09,947 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:10,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:11,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:12,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:13,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:13,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:14,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:14,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:14,465 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:14,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:14,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:15,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:18,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:20,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:20,770 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:20,770 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:20,776 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:20,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:21,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:23,776 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:25,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:25,703 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:25,703 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:25,709 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:25,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:27,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:28,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:28,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:28,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:29,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:32,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:32,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:32,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:32,734 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:32,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:33,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:34,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:38,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:38,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:38,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:38,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:38,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:39,330 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:42:39,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:40,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:43,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:43,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:43,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:43,944 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:43,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:43,945 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:43,950 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:44,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:45,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:45,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:48,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:48,364 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:48,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:48,370 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:48,864 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:49,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:51,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:54,956 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:54,956 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:54,957 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:54,963 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:55,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:55,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:56,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:58,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:58,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:58,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:00,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:00,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:00,476 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:00,482 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:00,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:01,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:02,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:04,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:04,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:04,969 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:04,975 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:05,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:05,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:07,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:09,329 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:43:09,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:09,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:09,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:09,919 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:10,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:11,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:13,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:13,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:13,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:13,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:14,087 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:14,088 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:14,088 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:14,094 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:14,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:15,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:19,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:20,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:20,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:20,930 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:20,936 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:21,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:21,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:24,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:25,033 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:25,034 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:25,034 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:25,040 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:25,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:25,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:28,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:28,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:28,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:30,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:30,285 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:30,285 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:30,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:30,291 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:30,917 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:31,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:35,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:35,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:35,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:35,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:35,454 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:35,924 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:37,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:39,334 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:43:40,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:41,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:41,191 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:41,191 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:41,197 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:41,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:43,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:43,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:43,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:43,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:45,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:49,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:49,838 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:49,838 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:49,844 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:49,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:50,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:51,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:54,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:54,111 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:54,111 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:54,117 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:54,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:55,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:56,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:58,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:58,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:58,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:59,272 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:59,273 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:59,273 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:59,279 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:59,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:01,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:01,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:06,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:06,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:06,666 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:06,666 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:06,672 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:06,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:07,967 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:09,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:44:11,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:11,746 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:11,746 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:11,747 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:11,752 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:11,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:13,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:13,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:13,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:13,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:16,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:16,139 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:16,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:16,145 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:16,976 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:17,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:17,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:22,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:25,084 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:25,085 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:25,085 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:25,091 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:25,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:25,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:28,091 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:28,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:28,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:28,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:29,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:29,003 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:29,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:29,009 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:29,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:29,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:34,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:34,744 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:34,745 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:34,745 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:34,750 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:34,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:36,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:39,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:39,243 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:39,243 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:39,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:39,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:39,338 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:44:40,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:42,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:43,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:43,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:43,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:44,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:44,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:44,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:44,184 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:45,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:45,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:46,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:48,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:48,839 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:48,839 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:48,846 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:49,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:50,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:50,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:53,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:53,101 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:53,101 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:53,107 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:54,025 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:54,025 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:56,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:58,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:58,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:58,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:59,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:59,201 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:59,201 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:59,207 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:00,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:00,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:01,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:05,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:05,123 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:05,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:05,130 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:06,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:06,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:07,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:09,337 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:45:10,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:10,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:10,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:10,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:11,042 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:12,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:12,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:13,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:13,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:13,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:15,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:15,513 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:15,513 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:15,519 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:16,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:17,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:18,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:20,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:20,990 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:20,990 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:20,996 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:21,051 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:22,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:22,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:27,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:27,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:27,873 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:27,878 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:28,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:28,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:28,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:28,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:28,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:30,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:32,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:32,638 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:32,638 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:32,643 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:33,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:33,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:34,069 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:37,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:37,958 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:37,958 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:37,971 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:38,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:38,972 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:39,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:45:40,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:43,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:43,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:43,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:43,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:43,601 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:43,601 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:43,607 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:44,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:44,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:46,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:49,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:50,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:50,660 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:50,660 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:50,666 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:51,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:52,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:54,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:54,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:54,891 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:54,891 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:54,896 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:55,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:56,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:58,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:58,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:58,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:59,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:59,169 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:59,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:59,175 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:00,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:00,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:00,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:03,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:03,404 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:03,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:03,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:04,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:05,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:06,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:07,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:07,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:07,502 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:07,508 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:08,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:09,342 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:46:10,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:11,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:11,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:11,802 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:11,802 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:11,808 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:12,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:13,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:13,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:14,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:16,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:21,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:21,735 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:21,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:21,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:21,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:22,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:24,128 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:26,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:28,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:28,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:28,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:28,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:28,598 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:28,598 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:28,604 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:29,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:30,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:32,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:32,751 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:32,752 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:32,752 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:32,758 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:33,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:34,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:37,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:37,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:37,023 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:37,028 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:37,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:38,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:38,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:39,344 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:46:42,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:42,029 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:42,029 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:42,035 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:42,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:43,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:43,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:43,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:43,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:44,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:46,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:46,362 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:46,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:46,368 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:47,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:48,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:48,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:51,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:51,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:51,279 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:51,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:52,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:52,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:54,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:55,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:55,939 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:55,939 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:55,945 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:56,167 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:58,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:58,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:58,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:58,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:59,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:00,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:00,412 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:00,412 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:00,418 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:01,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:02,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:05,454 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:06,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:06,129 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:06,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:06,135 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:06,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:08,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:09,350 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:47:11,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:11,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:11,419 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:11,419 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:11,425 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:12,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:13,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:14,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:15,723 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:15,724 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:15,724 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:15,730 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:16,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:16,765 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:18,195 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:20,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:20,345 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:20,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:20,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:21,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:22,202 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:22,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:25,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:25,694 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:25,694 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:25,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:26,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:27,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:28,209 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:28,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:28,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:28,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:31,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:31,476 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:31,476 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:31,482 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:32,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:33,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:34,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:36,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:36,400 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:36,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:36,407 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:37,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:38,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:39,348 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:47:39,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:41,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:41,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:41,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:41,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:42,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:43,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:43,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:43,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:44,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:44,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:46,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:46,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:46,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:46,268 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:47,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:48,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:50,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:50,591 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:50,592 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:50,592 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:50,599 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:51,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:52,238 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:55,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:55,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:55,862 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:55,862 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:55,868 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:56,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:58,245 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:58,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:58,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:58,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:00,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:00,301 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:00,301 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:00,307 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:01,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:01,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:02,250 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:04,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:04,859 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:04,859 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:04,865 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:05,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:06,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:06,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:09,353 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:48:09,919 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:09,920 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:09,920 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:09,926 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:10,256 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:11,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:12,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:13,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:15,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:15,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:15,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:15,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:16,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:17,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:18,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:22,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:22,193 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:22,193 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:22,199 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:22,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:23,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:24,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:26,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:26,462 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:26,462 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:26,468 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:27,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:28,283 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:28,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:28,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:28,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:28,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:32,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:32,763 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:32,763 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:32,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:33,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:33,805 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:34,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:37,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:37,347 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:37,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:37,353 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:38,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:38,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:39,353 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:48:39,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:42,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:42,714 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:42,714 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:42,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:43,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:43,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:43,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:43,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:44,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:44,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:47,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:47,101 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:47,101 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:47,109 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:47,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:48,307 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:50,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:51,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:51,637 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:51,637 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:51,643 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:52,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:54,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:55,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:56,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:56,447 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:56,448 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:56,453 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:57,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:58,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:58,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:58,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:58,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:00,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:01,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:01,578 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:01,579 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:01,585 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:02,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:04,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:06,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:06,142 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:06,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:06,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:06,147 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:06,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:08,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:09,358 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:49:11,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:11,760 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:11,761 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:11,761 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:11,766 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:12,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:13,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:13,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:14,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:16,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:20,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:20,622 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:20,622 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:20,628 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:21,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:21,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:22,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:26,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:27,804 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:27,805 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:27,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:27,811 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:28,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:28,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:28,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:28,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:30,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:32,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:32,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:32,662 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:32,662 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:32,669 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:33,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:34,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:36,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:36,629 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:36,629 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:36,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:37,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:37,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:38,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:39,357 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:49:43,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:43,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:43,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:43,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:43,454 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:43,597 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:43,598 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:43,604 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:44,372 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:46,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:48,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:50,908 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:50,909 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:50,909 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:50,915 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:51,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:52,382 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:53,916 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:57,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:57,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:57,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:57,244 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:57,385 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:58,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:58,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:58,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:58,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:59,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:02,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:02,874 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:02,874 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:02,880 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:03,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:04,396 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:04,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:08,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:08,304 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:08,304 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:08,310 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:08,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:09,359 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:50:10,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:10,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:12,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:12,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:12,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:12,178 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:12,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:13,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:13,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:13,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:14,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:15,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:16,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:16,698 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:16,699 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:16,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:17,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:18,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:20,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:22,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:22,228 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:22,228 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:22,234 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:22,420 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:24,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:26,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:27,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:27,568 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:27,569 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:27,574 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:28,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:28,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:28,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:28,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:28,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:31,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:32,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:32,039 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:32,040 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:32,046 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:32,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:34,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:37,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:37,971 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:37,975 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:37,975 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:37,981 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:38,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:39,362 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:50:40,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:42,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:43,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:43,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:43,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:46,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:46,477 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:46,478 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:46,483 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:47,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:47,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:48,449 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:52,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:52,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:52,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:52,081 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:52,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:53,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:54,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:58,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:58,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:58,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:58,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:00,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:00,542 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:00,542 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:00,548 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:01,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:02,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:03,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:04,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:04,850 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:04,850 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:04,855 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:05,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:06,469 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:08,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:09,364 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:51:10,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:10,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:10,286 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:10,292 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:10,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:12,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:13,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:13,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:13,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:14,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:15,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:15,237 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:15,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:15,243 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:15,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:16,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:20,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:20,165 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:20,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:20,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:20,171 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:20,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:22,491 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:25,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:25,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:25,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:25,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:25,285 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:25,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:26,496 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:28,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:28,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:28,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:29,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:29,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:29,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:29,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:29,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:30,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:30,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:35,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:35,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:35,649 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:35,649 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:35,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:36,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:36,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:39,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:51:41,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:42,606 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:42,607 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:42,607 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:42,614 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:43,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:43,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:43,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:43,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:44,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:46,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:47,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:47,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:47,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:47,014 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:47,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:48,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:52,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:55,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:55,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:55,474 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:55,480 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:55,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:56,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:57,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:58,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:58,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:58,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:00,826 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:00,827 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:00,827 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:00,833 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:01,536 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:02,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:02,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:04,991 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:04,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:04,992 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:04,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:05,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:06,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:07,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:09,368 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:52:10,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:10,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:10,964 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:10,970 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:11,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:12,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:13,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:13,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:13,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:13,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:15,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:15,496 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:15,496 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:15,502 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:15,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:16,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:19,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:24,055 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:24,056 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:24,056 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:24,062 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:24,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:25,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:26,567 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:28,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:28,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:28,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:30,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:30,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:30,898 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:30,898 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:30,904 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:31,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:32,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:34,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:34,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:34,949 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:34,955 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:35,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:35,956 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:36,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:39,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:52:41,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:41,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:41,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:41,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:41,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:41,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:42,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:43,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:43,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:43,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:46,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:46,321 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:46,321 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:46,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:46,327 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:46,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:48,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:51,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:51,134 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:51,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:51,140 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:51,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:52,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:52,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:57,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:58,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:58,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:58,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:00,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:00,360 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:00,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:00,366 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:00,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:02,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:02,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:05,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:05,355 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:05,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:05,361 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:05,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:06,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:08,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:09,372 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:53:09,656 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:09,657 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:09,657 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:09,663 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:10,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:10,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:13,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:13,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:13,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:13,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:14,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:14,546 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:14,546 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:14,552 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:14,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:16,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:18,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:18,696 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:18,696 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:18,696 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:18,702 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:19,628 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:20,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:23,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:23,334 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:23,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:23,340 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:23,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:24,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:24,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:27,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:27,645 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:27,645 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:27,652 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:28,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:28,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:28,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:28,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:28,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:29,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:32,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:32,359 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:32,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:32,366 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:32,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:34,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:35,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:37,219 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:37,220 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:37,220 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:37,226 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:37,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:38,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:39,375 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:53:41,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:42,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:42,098 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:42,098 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:42,104 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:42,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:43,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:43,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:43,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:44,663 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:46,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:46,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:46,494 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:46,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:46,500 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:46,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:48,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:51,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:53,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:53,963 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:53,963 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:53,969 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:54,672 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:56,677 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:56,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:58,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:58,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:58,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:59,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:59,543 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:59,543 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:59,549 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:59,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:00,682 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:02,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:05,011 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:05,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:05,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:05,018 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:05,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:06,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:08,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:09,377 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:54:09,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:09,698 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:09,698 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:09,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:10,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:10,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:13,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:13,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:13,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:13,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:14,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:14,530 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:14,531 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:14,536 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:14,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:16,701 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:19,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:19,089 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:19,089 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:19,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:19,096 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:19,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:20,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:23,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:23,365 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:23,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:23,371 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:23,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:24,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:24,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:28,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:28,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:28,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:28,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:28,657 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:28,658 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:28,664 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:28,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:29,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:30,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:33,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:33,346 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:33,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:33,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:33,722 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:34,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:35,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:38,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:38,021 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:38,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:38,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:38,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:39,379 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:54:40,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:41,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:43,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:43,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:43,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:44,519 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:44,519 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:44,519 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:44,525 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:44,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:46,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:46,738 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:49,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:49,402 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:49,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:49,408 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:49,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:50,743 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:52,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:53,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:53,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:53,639 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:53,645 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:53,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:54,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:57,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:58,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:58,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:58,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:59,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:59,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:59,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:59,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:59,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:00,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:03,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:05,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:05,593 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:05,593 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:05,599 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:05,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:06,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:08,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:09,385 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:55:11,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:11,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:11,515 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:11,520 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:11,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:12,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:13,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:13,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:13,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:14,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:16,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:16,200 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:16,200 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:16,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:16,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:18,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:20,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:20,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:20,835 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:20,835 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:20,840 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:21,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:22,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:25,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:26,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:26,042 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:26,042 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:26,047 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:26,786 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:28,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:28,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:28,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:28,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:31,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:33,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:33,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:33,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:33,159 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:33,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:34,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:37,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:37,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:37,392 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:37,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:37,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:37,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:38,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:39,384 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:55:41,920 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:41,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:41,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:41,927 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:42,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:42,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:42,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:43,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:43,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:48,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:48,036 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:48,036 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:48,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:48,042 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:48,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:50,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:52,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:52,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:52,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:52,457 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:52,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:53,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:54,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:57,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:57,224 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:57,224 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:57,229 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:57,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:58,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:58,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:58,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:58,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:58,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:02,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:02,573 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:02,573 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:02,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:02,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:04,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:04,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:07,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:07,017 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:07,017 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:07,023 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:07,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:08,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:09,389 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:56:10,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:13,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:13,198 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:13,198 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:13,205 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:13,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:13,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:13,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:13,849 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:14,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:15,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:18,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:18,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:18,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:18,930 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:19,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:20,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:20,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:23,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:23,174 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:23,174 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:23,180 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:23,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:24,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:26,180 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:28,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:28,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:28,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:31,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:31,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:31,625 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:31,625 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:31,631 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:31,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:32,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:36,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:38,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:38,426 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:38,427 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:38,433 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:38,878 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:39,388 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:56:40,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:42,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:42,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:42,672 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:42,673 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:42,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:42,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:43,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:43,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:44,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:47,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:51,062 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:51,062 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:51,063 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:51,069 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:51,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:52,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:53,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:55,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:55,742 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:55,742 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:55,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:55,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:56,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:58,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:58,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:58,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:58,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:59,879 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:59,880 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:59,880 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:59,886 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:59,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:00,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:03,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:05,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:05,035 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:05,035 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:05,041 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:05,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:06,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:09,042 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:09,390 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:57:10,191 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:10,192 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:10,192 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:10,197 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:10,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:12,919 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:13,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:13,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:13,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:14,690 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:15,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:15,629 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:15,629 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:15,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:15,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:16,925 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:20,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:21,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:21,345 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:21,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:21,351 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:21,928 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:22,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:26,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:26,078 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:26,078 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:26,078 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:26,084 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:26,936 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:28,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:28,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:28,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:28,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:31,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:33,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:33,839 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:33,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:33,846 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:33,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:34,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:36,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:38,078 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:38,079 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:38,079 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:38,085 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:38,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:38,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:39,393 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:57:42,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:42,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:42,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:42,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:42,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:42,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:43,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:44,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:47,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:50,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:50,750 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:50,750 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:50,756 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:50,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:52,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:52,967 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:57,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:57,080 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:57,080 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:57,086 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:57,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:58,086 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:58,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:58,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:58,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:58,974 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:01,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:01,554 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:01,554 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:01,560 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:01,976 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:02,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:03,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:06,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:06,296 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:06,297 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:06,303 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:06,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:08,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:09,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:09,395 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:58:11,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:11,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:11,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:11,304 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:11,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:12,992 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:13,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:13,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:13,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:14,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:16,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:16,091 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:16,091 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:16,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:16,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:16,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:20,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:22,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:22,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:22,020 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:22,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:23,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:23,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:26,027 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:27,715 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:27,716 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:27,716 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:27,722 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:28,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:28,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:28,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:28,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:29,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:31,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:31,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:31,961 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:31,961 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:31,967 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:32,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:33,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:36,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:36,374 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:36,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:36,381 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:37,024 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:37,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:39,029 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:39,397 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:58:40,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:40,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:40,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:40,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:41,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:42,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:43,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:43,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:43,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:45,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:45,015 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:45,016 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:45,030 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:45,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:47,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:48,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:49,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:49,842 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:49,842 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:49,849 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:50,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:51,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:53,850 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:55,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:55,049 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:55,050 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:55,056 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:55,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:57,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:58,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:58,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:58,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:59,654 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:59,731 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:59,731 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:59,732 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:59,737 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:00,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:01,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:04,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:04,173 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:04,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:04,179 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:05,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:05,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:05,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:09,399 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:59:10,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:11,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:11,326 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:11,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:11,332 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:12,077 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:13,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:13,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:13,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:13,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:15,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:16,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:16,670 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:16,670 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:16,676 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:17,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:19,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:20,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:22,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:22,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:22,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:22,573 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:23,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:25,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:26,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:27,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:27,166 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:27,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:27,172 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:28,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:28,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:28,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:28,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:29,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:31,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:33,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:33,385 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:33,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:33,391 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:34,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:35,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:37,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:38,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:38,326 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:38,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:38,332 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:39,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:39,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:59:41,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:42,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:42,614 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:42,614 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:42,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:42,620 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:43,120 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:43,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:45,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:47,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:47,198 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:47,198 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:47,204 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:48,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:48,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:49,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:51,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:51,891 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:51,891 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:51,897 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:52,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:53,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:53,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:56,078 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:56,079 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:56,079 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:56,085 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:56,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:57,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:58,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:58,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:58,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:59,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:00,604 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:00,604 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:00,605 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:00,610 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:01,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:03,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:04,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:04,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:04,887 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:04,887 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:04,893 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:05,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:07,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:08,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:08,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:08,871 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:08,876 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:09,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:09,404 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:00:10,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:11,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:13,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:13,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:13,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:15,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:17,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:17,282 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:17,282 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:17,288 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:18,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:19,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:21,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:22,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:22,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:22,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:22,173 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:22,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:23,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:27,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:27,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:27,345 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:27,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:27,351 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:28,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:28,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:28,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:28,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:29,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:32,292 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:32,293 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:32,293 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:32,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:32,299 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:33,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:35,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:36,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:36,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:36,613 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:36,619 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:37,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:37,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:39,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:39,409 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:00:41,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:41,371 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:41,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:41,376 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:42,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:43,210 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:43,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:43,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:43,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:47,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:47,680 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:47,680 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:47,689 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:48,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:48,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:49,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:52,098 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:52,099 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:52,099 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:52,105 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:52,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:53,225 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:54,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:57,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:57,185 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:57,186 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:57,191 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:57,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:58,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:58,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:58,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:59,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:59,744 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:01,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:01,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:01,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:01,733 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:02,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:03,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:04,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:06,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:06,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:06,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:06,716 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:07,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:09,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:09,408 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:01:10,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:13,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:13,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:13,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:13,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:13,682 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:13,683 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:13,689 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:14,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:15,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:15,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:18,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:18,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:18,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:18,889 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:19,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:20,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:21,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:23,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:23,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:23,475 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:23,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:24,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:25,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:26,482 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:28,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:28,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:28,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:31,419 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:31,419 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:31,420 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:31,425 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:32,279 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:32,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:33,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:35,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:35,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:35,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:35,890 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:36,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:37,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:37,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:39,410 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:01:40,959 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:40,960 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:40,960 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:40,966 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:41,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:42,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:43,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:43,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:43,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:45,823 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:45,823 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:45,824 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:45,829 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:46,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:47,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:48,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:52,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:52,369 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:52,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:52,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:53,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:53,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:54,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:57,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:57,301 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:57,301 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:57,307 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:57,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:58,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:58,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:58,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:59,323 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:59,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:01,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:01,578 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:01,579 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:01,585 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:02,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:03,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:05,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:06,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:06,074 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:06,074 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:06,080 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:06,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:07,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:09,412 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:02:10,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:10,451 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:10,451 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:10,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:11,338 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:11,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:13,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:13,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:13,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:13,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:14,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:14,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:14,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:14,740 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:15,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:16,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:17,349 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:20,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:20,666 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:20,666 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:20,672 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:21,353 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:22,672 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:23,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:24,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:24,986 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:24,986 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:24,991 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:25,362 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:27,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:27,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:28,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:28,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:28,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:29,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:29,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:29,288 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:29,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:29,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:31,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:33,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:37,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:37,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:37,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:37,763 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:38,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:38,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:39,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:39,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:02:41,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:41,602 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:41,602 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:41,608 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:42,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:43,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:43,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:43,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:44,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:46,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:46,279 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:46,279 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:46,285 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:46,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:47,395 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:50,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:50,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:50,601 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:50,602 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:50,608 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:51,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:53,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:55,180 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:55,181 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:55,181 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:55,187 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:55,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:56,188 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:57,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:58,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:58,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:58,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:00,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:00,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:00,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:00,581 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:01,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:01,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:03,422 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:05,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:05,581 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:05,581 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:05,588 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:06,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:07,428 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:07,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:09,417 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:03:09,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:09,858 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:09,858 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:09,865 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:10,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:11,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:12,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:13,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:13,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:13,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:18,508 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:18,508 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:18,508 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:18,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:18,514 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:19,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:21,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:23,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:24,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:24,879 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:24,880 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:24,886 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:25,449 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:27,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:28,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:28,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:28,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:28,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:29,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:29,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:29,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:29,401 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:29,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:31,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:34,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:34,142 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:34,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:34,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:34,148 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:34,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:35,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:39,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:39,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:39,401 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:39,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:39,407 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:39,419 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:03:39,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:41,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:43,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:43,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:43,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:44,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:44,159 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:44,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:44,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:44,166 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:44,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:45,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:48,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:48,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:48,024 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:48,030 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:48,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:49,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:50,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:53,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:53,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:53,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:53,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:53,488 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:55,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:55,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:58,263 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:58,264 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:58,265 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:58,271 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:58,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:58,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:58,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:58,496 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:59,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:00,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:05,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:05,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:05,299 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:05,305 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:05,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:06,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:07,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:09,425 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:04:09,776 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:09,777 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:09,777 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:09,783 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:10,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:11,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:11,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:13,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:13,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:13,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:14,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:14,074 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:14,074 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:14,080 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:14,521 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:15,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:17,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:18,654 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:18,655 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:18,655 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:18,661 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:19,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:21,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:22,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:23,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:23,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:23,976 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:23,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:24,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:25,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:27,984 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:28,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:28,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:28,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:28,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:28,588 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:28,588 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:28,594 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:29,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:31,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:33,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:33,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:33,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:33,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:33,304 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:33,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:35,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:38,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:39,423 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:04:41,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:41,803 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:41,804 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:41,810 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:42,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:43,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:43,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:43,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:43,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:45,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:45,846 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:45,846 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:45,852 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:46,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:47,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:48,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:50,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:50,635 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:50,635 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:50,642 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:51,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:53,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:54,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:58,067 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:58,068 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:58,068 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:58,074 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:58,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:58,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:58,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:58,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:59,588 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:00,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:02,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:02,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:02,195 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:02,201 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:02,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:03,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:06,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:07,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:07,646 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:07,646 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:07,652 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:08,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:09,428 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:05:09,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:11,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:13,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:13,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:13,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:15,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:15,089 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:15,089 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:15,096 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:15,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:17,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:17,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:20,606 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:20,607 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:20,607 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:20,613 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:20,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:21,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:22,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:25,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:25,146 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:25,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:25,152 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:25,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:27,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:28,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:28,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:28,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:28,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:29,811 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:29,811 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:29,812 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:29,817 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:30,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:31,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:33,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:35,086 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:35,086 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:35,087 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:35,093 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:35,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:37,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:38,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:38,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:38,976 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:38,976 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:38,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:39,427 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:05:39,648 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:41,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:43,283 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:43,285 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:43,285 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:43,291 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:43,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:43,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:43,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:44,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:45,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:48,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:48,631 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:48,631 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:48,638 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:48,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:49,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:51,669 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:52,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:52,863 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:52,863 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:52,869 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:53,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:54,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:55,675 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:58,070 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:58,071 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:58,071 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:58,077 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:58,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:58,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:58,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:58,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:59,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:00,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:05,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:05,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:05,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:05,560 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:05,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:06,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:07,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:09,429 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:06:12,431 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:12,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:12,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:12,534 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:12,541 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:12,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:13,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:13,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:13,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:13,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:17,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:17,732 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:17,737 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:17,737 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:17,743 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:18,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:19,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:22,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:22,023 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:22,023 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:22,029 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:22,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:23,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:23,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:28,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:28,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:28,159 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:28,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:28,165 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:28,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:28,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:28,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:28,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:29,724 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:32,667 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:32,668 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:32,668 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:32,674 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:32,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:33,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:35,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:38,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:39,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:06:40,950 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:40,951 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:40,951 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:40,957 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:41,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:43,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:43,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:43,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:43,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:44,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:46,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:46,588 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:46,589 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:46,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:46,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:47,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:50,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:51,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:51,281 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:51,281 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:51,288 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:51,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:53,759 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:56,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:58,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:58,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:58,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:59,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:59,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:59,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:59,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:59,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:01,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:01,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:06,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:07,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:07,387 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:07,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:07,393 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:07,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:09,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:07:09,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:12,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:12,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:12,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:12,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:12,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:12,784 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:13,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:13,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:13,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:13,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:17,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:19,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:19,135 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:19,135 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:19,141 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:19,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:21,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:23,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:24,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:24,902 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:24,902 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:24,908 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:25,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:27,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:28,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:28,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:28,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:28,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:29,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:29,431 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:29,431 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:29,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:29,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:31,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:34,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:35,262 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:35,263 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:35,263 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:35,269 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:35,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:37,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:39,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:07:40,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:40,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:40,621 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:40,622 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:40,628 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:40,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:40,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:43,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:43,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:43,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:44,940 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:44,941 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:44,941 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:44,947 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:45,834 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:45,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:46,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:49,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:49,889 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:49,889 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:49,896 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:50,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:50,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:51,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:56,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:58,231 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:58,232 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:58,232 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:58,239 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:58,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:58,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:58,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:58,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:58,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:02,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:03,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:03,052 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:03,052 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:03,058 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:03,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:04,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:08,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:08,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:08,546 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:08,546 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:08,551 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:08,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:08,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:09,438 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:08:13,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:13,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:13,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:13,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:14,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:14,338 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:14,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:14,344 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:14,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:14,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:19,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:22,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:22,191 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:22,191 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:22,197 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:22,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:22,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:25,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:27,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:27,283 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:27,283 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:27,289 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:27,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:28,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:28,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:28,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:28,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:30,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:32,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:32,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:32,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:32,268 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:32,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:32,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:36,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:38,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:38,577 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:38,577 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:38,583 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:38,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:38,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:39,444 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:08:41,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:43,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:43,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:43,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:43,609 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:43,614 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:43,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:44,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:46,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:48,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:48,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:48,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:48,304 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:48,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:48,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:52,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:52,229 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:52,229 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:52,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:52,235 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:52,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:52,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:57,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:58,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:58,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:58,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:58,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:58,782 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:58,782 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:58,788 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:58,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:58,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:02,826 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:04,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:04,198 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:04,198 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:04,204 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:04,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:04,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:08,204 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:08,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:08,670 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:08,671 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:08,677 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:08,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:08,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:09,443 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:09:12,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:12,833 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:12,833 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:12,839 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:12,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:12,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:13,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:13,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:13,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:13,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:17,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:17,253 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:17,253 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:17,259 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:17,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:18,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:19,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:21,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:21,502 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:21,503 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:21,508 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:21,974 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:22,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:24,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:28,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:28,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:28,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:29,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:30,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:30,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:30,612 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:30,618 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:30,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:30,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:34,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:38,936 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:38,937 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:38,937 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:38,943 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:38,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:39,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:09:40,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:41,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:43,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:45,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:47,230 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:47,231 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:47,231 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:47,237 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:48,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:49,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:51,273 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:51,751 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:51,751 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:51,751 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:51,757 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:52,019 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:53,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:56,758 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:58,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:58,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:58,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:58,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:58,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:58,609 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:58,614 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:59,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:59,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:02,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:04,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:04,332 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:04,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:04,339 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:05,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:05,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:08,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:09,447 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:10:09,724 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:09,725 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:09,725 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:09,730 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:10,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:11,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:13,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:13,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:13,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:13,659 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:15,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:15,086 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:15,086 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:15,092 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:16,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:17,057 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:19,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:20,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:20,692 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:20,692 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:20,698 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:21,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:21,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:24,699 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:25,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:25,690 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:25,690 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:25,696 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:26,069 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:27,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:28,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:28,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:28,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:30,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:32,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:32,173 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:32,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:32,179 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:33,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:33,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:36,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:37,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:37,059 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:37,059 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:37,065 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:37,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:39,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:39,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:10:41,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:42,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:42,587 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:42,587 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:42,593 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:43,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:43,096 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:43,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:43,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:46,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:48,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:48,101 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:48,101 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:48,106 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:49,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:49,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:52,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:53,712 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:53,713 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:53,713 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:53,719 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:54,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:55,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:57,720 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:58,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:58,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:58,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:00,076 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:00,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:00,077 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:00,083 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:00,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:01,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:03,118 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:04,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:04,383 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:04,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:04,389 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:05,128 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:05,128 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:08,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:09,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:11:09,987 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:09,988 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:09,988 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:09,994 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:10,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:11,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:13,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:13,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:13,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:13,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:15,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:15,754 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:15,754 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:15,760 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:16,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:17,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:18,797 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:21,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:21,169 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:21,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:21,174 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:22,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:23,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:24,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:25,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:25,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:25,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:25,930 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:26,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:27,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:28,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:28,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:28,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:29,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:30,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:30,123 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:30,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:30,130 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:30,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:31,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:35,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:36,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:36,395 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:36,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:36,401 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:37,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:37,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:39,453 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:11:40,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:41,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:41,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:41,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:41,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:41,182 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:43,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:43,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:43,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:43,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:45,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:45,572 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:45,572 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:45,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:45,578 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:46,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:47,192 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:50,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:51,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:51,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:51,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:51,203 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:52,196 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:53,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:56,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:58,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:58,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:58,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:59,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:59,711 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:59,711 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:59,717 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:00,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:01,210 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:01,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:03,815 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:03,816 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:03,816 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:03,822 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:04,213 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:05,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:06,823 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:08,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:08,268 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:08,268 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:08,273 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:09,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:09,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:09,459 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:12:12,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:13,369 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:13,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:13,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:13,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:13,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:13,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:14,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:15,231 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:17,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:21,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:21,825 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:21,825 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:21,830 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:22,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:22,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:23,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:27,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:28,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:28,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:28,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:29,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:29,008 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:29,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:29,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:29,251 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:29,251 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:33,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:34,060 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:34,061 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:34,061 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:34,067 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:34,256 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:35,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:38,067 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:39,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:12:39,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:39,493 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:39,493 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:39,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:40,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:41,269 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:43,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:43,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:43,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:43,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:48,095 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:48,096 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:48,096 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:48,103 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:48,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:49,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:49,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:54,031 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:54,032 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:54,033 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:54,038 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:54,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:55,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:55,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:58,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:58,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:58,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:59,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:59,982 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:59,982 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:59,987 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:00,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:01,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:01,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:04,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:04,855 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:04,856 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:04,861 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:05,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:05,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:06,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:08,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:08,998 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:08,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:09,004 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:09,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:09,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:09,463 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:13:12,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:13,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:13,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:13,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:13,480 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:13,672 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:13,673 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:13,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:14,316 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:15,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:17,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:18,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:18,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:18,196 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:18,201 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:18,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:19,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:23,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:23,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:23,995 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:23,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:24,002 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:24,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:25,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:28,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:28,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:28,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:28,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:28,679 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:28,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:28,679 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:28,685 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:29,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:29,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:33,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:34,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:34,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:34,534 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:34,540 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:35,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:35,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:39,462 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:13:39,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:39,868 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:39,869 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:39,869 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:39,875 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:40,355 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:41,357 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:43,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:43,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:43,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:44,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:44,860 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:44,861 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:44,861 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:44,867 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:45,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:45,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:49,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:51,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:51,682 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:51,682 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:51,688 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:52,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:53,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:55,688 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:56,987 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:56,988 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:56,988 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:56,994 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:57,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:57,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:58,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:58,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:58,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:01,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:01,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:01,196 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:01,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:01,202 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:01,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:03,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:06,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:09,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:14:09,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:09,523 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:09,523 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:09,529 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:10,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:11,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:11,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:13,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:13,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:13,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:16,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:16,249 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:16,249 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:16,255 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:16,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:17,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:17,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:20,486 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:20,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:20,487 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:20,492 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:21,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:21,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:22,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:26,728 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:26,729 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:26,729 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:26,735 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:27,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:27,736 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:28,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:28,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:28,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:29,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:31,266 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:31,270 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:31,270 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:31,276 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:31,428 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:33,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:33,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:36,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:36,400 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:36,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:36,405 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:36,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:37,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:38,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:39,467 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:14:40,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:40,360 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:40,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:40,366 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:40,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:41,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:43,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:43,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:43,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:43,683 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:46,230 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:46,230 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:46,230 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:46,236 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:46,450 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:47,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:49,273 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:51,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:51,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:51,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:51,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:51,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:53,462 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:55,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:56,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:56,162 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:56,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:56,168 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:56,465 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:57,469 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:58,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:58,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:58,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:00,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:00,614 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:00,615 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:00,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:00,620 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:01,474 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:01,474 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:04,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:04,863 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:04,863 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:04,869 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:05,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:05,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:07,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:09,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:09,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:09,195 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:09,201 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:09,469 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:15:09,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:11,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:11,488 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:13,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:13,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:13,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:15,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:15,858 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:15,858 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:15,864 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:16,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:16,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:17,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:21,902 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:22,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:22,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:22,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:22,183 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:22,503 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:23,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:27,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:27,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:27,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:27,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:27,776 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:28,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:28,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:28,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:28,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:29,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:32,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:33,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:33,446 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:33,446 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:33,452 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:33,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:35,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:38,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:39,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:39,167 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:39,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:39,173 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:39,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:15:39,527 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:41,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:43,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:43,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:43,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:43,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:43,625 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:43,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:43,625 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:43,632 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:44,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:45,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:48,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:49,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:49,115 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:49,115 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:49,121 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:49,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:51,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:54,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:55,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:55,445 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:55,445 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:55,451 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:55,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:57,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:58,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:58,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:58,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:59,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:59,602 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:59,603 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:59,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:59,608 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:00,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:01,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:03,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:03,661 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:03,661 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:03,667 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:04,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:04,668 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:05,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:09,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:16:10,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:10,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:10,611 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:10,611 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:10,617 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:11,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:11,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:13,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:13,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:13,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:15,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:15,373 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:15,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:15,378 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:15,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:16,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:17,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:20,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:20,251 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:20,251 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:20,258 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:20,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:21,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:22,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:24,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:24,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:24,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:24,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:24,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:25,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:28,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:28,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:28,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:28,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:29,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:29,037 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:29,037 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:29,043 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:29,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:31,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:33,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:33,187 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:33,187 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:33,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:33,192 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:33,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:35,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:38,193 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:38,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:38,661 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:38,662 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:38,667 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:39,479 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:16:39,617 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:39,617 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:43,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:43,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:43,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:43,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:43,626 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:43,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:43,627 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:43,632 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:44,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:45,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:47,979 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:47,981 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:47,981 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:47,986 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:48,628 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:48,987 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:49,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:52,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:52,535 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:52,535 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:52,542 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:52,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:53,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:54,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:57,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:57,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:57,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:57,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:57,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:58,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:58,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:58,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:59,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:59,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:02,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:02,344 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:02,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:02,350 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:02,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:03,652 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:05,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:09,478 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:17:10,479 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:11,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:11,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:11,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:11,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:11,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:13,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:13,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:13,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:13,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:15,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:16,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:16,204 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:16,204 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:16,210 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:16,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:17,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:21,246 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:24,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:24,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:24,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:24,734 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:25,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:25,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:26,735 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:28,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:28,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:28,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:29,767 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:29,768 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:29,768 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:29,774 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:30,690 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:31,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:31,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:35,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:35,004 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:35,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:35,010 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:35,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:37,011 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:37,701 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:39,480 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:17:40,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:40,060 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:40,060 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:40,066 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:40,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:41,707 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:42,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:43,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:43,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:43,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:47,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:47,175 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:47,175 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:47,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:47,181 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:47,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:49,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:51,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:51,598 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:51,598 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:51,604 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:51,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:52,604 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:53,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:56,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:56,675 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:56,675 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:56,681 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:56,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:57,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:57,730 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:58,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:58,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:58,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:00,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:00,862 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:00,862 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:00,868 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:01,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:01,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:02,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:06,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:06,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:06,038 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:06,043 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:06,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:07,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:08,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:09,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:18:10,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:10,795 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:10,796 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:10,802 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:11,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:11,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:13,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:13,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:13,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:13,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:18,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:18,842 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:18,842 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:18,843 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:18,848 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:19,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:19,763 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:23,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:23,382 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:23,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:23,388 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:23,766 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:24,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:25,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:27,775 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:27,776 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:27,776 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:27,782 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:28,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:28,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:28,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:28,775 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:29,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:29,778 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:32,986 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:32,987 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:32,987 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:32,993 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:33,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:35,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:35,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:38,731 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:38,732 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:38,732 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:38,739 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:38,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:39,485 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:18:39,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:40,486 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:43,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:43,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:43,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:43,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:43,899 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:43,900 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:43,906 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:44,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:45,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:45,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:48,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:48,360 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:48,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:48,366 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:48,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:49,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:51,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:52,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:52,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:52,743 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:52,749 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:52,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:53,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:56,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:57,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:57,308 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:57,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:57,314 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:57,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:58,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:58,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:58,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:59,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:02,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:05,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:05,733 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:05,733 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:05,739 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:05,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:07,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:07,832 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:09,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:19:12,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:12,121 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:12,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:12,127 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:12,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:13,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:13,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:13,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:13,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:13,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:18,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:18,244 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:18,244 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:18,244 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:18,250 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:18,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:19,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:22,639 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:22,640 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:22,640 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:22,646 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:22,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:23,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:23,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:27,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:27,815 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:27,815 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:27,821 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:27,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:28,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:28,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:28,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:28,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:29,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:33,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:35,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:35,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:35,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:35,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:36,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:37,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:39,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:39,489 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:19:41,782 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:41,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:41,783 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:41,789 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:41,879 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:43,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:43,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:43,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:43,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:44,672 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:48,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:48,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:48,515 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:48,521 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:48,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:49,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:50,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:53,761 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:53,762 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:53,762 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:53,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:53,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:55,768 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:55,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:58,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:58,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:58,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:01,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:02,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:02,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:02,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:02,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:02,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:03,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:07,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:09,495 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:20:10,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:10,531 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:10,531 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:10,537 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:10,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:11,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:12,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:13,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:13,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:13,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:14,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:14,625 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:14,625 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:14,631 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:14,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:15,932 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:17,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:18,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:18,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:18,787 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:18,793 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:18,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:19,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:22,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:23,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:23,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:23,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:23,216 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:23,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:25,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:27,920 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:27,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:27,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:27,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:27,927 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:27,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:28,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:28,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:28,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:29,954 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:33,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:34,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:34,002 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:34,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:34,008 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:34,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:35,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:38,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:38,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:38,288 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:38,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:38,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:39,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:39,493 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:20:39,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:43,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:43,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:43,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:44,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:47,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:47,118 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:47,118 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:47,123 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:47,976 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:49,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:50,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:51,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:51,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:51,432 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:51,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:51,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:53,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:55,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:55,958 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:55,959 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:55,959 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:55,965 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:55,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:57,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:58,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:58,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:58,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:00,595 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:00,595 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:00,596 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:00,596 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:00,601 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:00,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:02,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:05,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:05,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:05,246 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:05,252 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:06,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:06,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:08,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:09,499 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:21:11,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:12,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:12,590 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:12,590 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:12,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:13,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:13,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:13,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:13,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:14,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:16,639 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:17,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:17,062 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:17,062 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:17,068 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:18,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:18,024 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:22,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:25,879 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:25,880 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:25,880 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:25,886 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:26,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:27,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:28,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:28,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:28,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:28,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:31,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:31,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:31,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:31,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:32,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:33,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:34,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:35,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:35,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:35,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:35,950 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:36,047 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:38,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:38,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:39,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:21:40,471 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:40,472 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:40,472 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:40,478 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:41,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:42,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:43,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:43,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:43,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:44,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:44,599 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:44,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:44,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:44,605 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:45,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:46,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:49,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:50,952 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:50,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:50,953 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:50,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:51,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:52,073 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:54,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:58,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:58,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:58,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:59,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:59,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:59,458 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:59,464 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:00,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:00,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:02,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:05,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:08,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:08,346 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:08,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:08,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:09,096 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:09,500 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:22:10,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:11,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:12,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:12,863 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:12,863 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:12,869 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:13,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:13,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:13,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:13,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:14,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:16,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:18,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:18,364 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:18,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:18,370 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:19,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:20,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:22,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:23,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:23,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:23,779 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:23,784 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:24,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:26,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:27,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:28,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:28,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:28,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:28,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:28,613 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:28,613 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:28,619 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:29,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:30,129 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:33,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:33,855 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:33,855 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:33,856 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:33,861 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:34,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:36,137 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:38,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:39,502 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:22:40,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:40,184 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:40,184 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:40,190 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:41,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:42,147 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:43,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:43,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:43,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:44,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:44,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:44,975 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:44,975 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:44,980 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:45,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:46,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:49,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:49,435 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:49,436 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:49,443 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:50,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:50,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:52,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:54,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:54,082 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:54,082 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:54,087 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:54,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:56,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:56,167 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:58,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:58,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:58,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:58,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:58,711 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:58,711 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:58,717 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:59,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:00,173 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:01,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:03,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:03,875 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:03,875 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:03,881 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:04,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:06,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:06,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:09,504 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:23:10,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:10,630 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:10,630 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:10,635 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:11,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:12,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:12,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:13,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:13,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:13,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:15,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:15,271 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:15,271 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:15,276 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:16,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:18,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:18,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:23,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:23,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:23,599 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:23,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:23,604 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:24,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:26,213 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:28,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:28,162 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:28,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:28,168 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:28,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:28,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:28,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:28,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:28,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:30,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:32,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:32,382 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:32,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:32,387 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:33,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:34,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:34,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:36,488 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:36,489 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:36,489 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:36,495 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:37,226 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:38,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:39,496 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:39,510 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:23:41,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:41,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:41,031 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:41,036 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:41,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:42,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:43,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:43,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:43,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:44,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:47,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:47,201 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:47,201 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:47,207 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:47,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:48,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:50,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:52,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:52,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:52,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:52,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:52,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:54,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:56,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:58,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:58,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:58,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:58,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:58,615 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:58,615 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:58,621 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:59,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:00,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:01,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:03,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:03,692 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:03,692 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:03,698 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:04,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:06,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:06,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:08,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:08,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:08,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:08,305 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:09,274 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:09,508 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:24:10,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:12,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:12,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:12,587 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:12,588 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:12,593 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:13,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:13,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:13,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:13,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:14,283 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:17,622 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:18,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:18,496 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:18,496 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:18,502 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:19,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:20,292 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:23,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:24,788 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:24,789 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:24,790 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:24,796 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:25,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:26,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:28,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:28,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:28,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:28,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:33,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:34,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:34,368 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:34,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:34,374 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:35,311 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:36,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:38,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:38,540 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:38,541 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:38,546 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:39,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:39,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:24:39,515 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:40,320 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:43,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:43,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:43,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:44,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:44,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:44,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:44,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:44,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:45,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:46,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:49,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:49,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:49,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:49,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:49,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:50,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:51,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:53,758 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:53,759 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:53,760 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:53,765 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:54,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:56,344 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:56,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:58,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:58,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:58,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:58,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:58,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:58,702 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:58,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:59,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:00,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:02,745 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:04,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:04,906 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:04,906 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:04,911 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:05,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:06,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:07,912 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:09,513 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:25:10,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:10,046 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:10,046 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:10,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:10,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:12,368 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:13,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:13,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:13,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:13,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:14,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:14,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:14,815 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:14,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:15,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:16,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:18,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:19,774 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:19,775 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:19,775 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:19,781 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:20,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:22,382 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:24,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:25,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:25,543 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:25,543 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:25,549 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:26,388 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:28,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:28,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:28,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:28,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:30,596 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:32,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:32,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:32,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:32,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:33,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:34,400 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:36,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:36,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:36,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:36,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:36,949 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:37,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:38,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:39,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:25:41,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:42,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:42,571 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:42,571 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:42,577 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:43,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:43,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:43,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:43,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:44,415 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:46,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:51,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:51,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:51,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:51,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:52,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:52,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:54,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:57,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:58,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:58,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:58,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:59,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:59,378 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:59,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:59,384 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:59,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:00,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:03,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:04,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:04,712 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:04,712 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:04,718 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:05,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:06,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:08,719 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:09,517 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:26:11,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:11,593 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:11,593 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:11,599 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:12,454 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:13,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:13,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:13,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:14,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:14,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:16,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:16,686 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:16,686 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:16,692 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:17,462 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:18,465 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:19,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:21,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:21,252 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:21,252 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:21,258 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:21,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:22,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:25,259 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:25,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:25,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:25,866 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:25,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:26,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:28,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:28,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:28,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:28,480 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:30,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:32,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:32,631 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:32,631 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:32,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:33,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:34,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:36,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:37,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:37,572 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:37,572 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:37,578 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:38,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:39,520 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:26:40,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:42,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:42,171 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:42,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:42,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:42,176 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:42,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:43,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:43,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:43,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:44,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:47,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:47,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:47,312 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:47,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:47,318 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:47,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:48,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:51,667 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:51,668 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:51,668 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:51,675 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:52,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:52,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:54,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:57,676 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:57,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:57,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:57,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:57,892 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:58,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:58,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:58,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:58,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:00,526 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:03,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:05,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:05,591 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:05,591 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:05,596 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:06,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:08,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:09,523 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:27:09,524 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:10,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:10,190 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:10,191 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:10,196 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:10,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:12,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:13,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:13,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:13,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:14,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:16,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:16,114 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:16,114 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:16,121 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:16,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:18,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:20,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:20,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:20,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:20,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:20,780 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:21,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:22,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:25,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:25,779 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:25,779 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:25,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:25,785 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:26,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:28,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:28,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:28,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:28,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:30,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:30,082 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:30,082 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:30,088 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:30,572 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:31,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:32,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:35,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:35,815 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:35,815 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:35,821 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:36,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:36,822 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:38,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:39,522 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:27:40,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:40,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:40,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:40,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:40,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:42,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:42,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:43,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:43,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:43,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:45,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:45,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:45,223 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:45,229 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:45,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:46,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:48,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:50,072 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:50,072 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:50,073 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:50,079 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:50,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:52,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:54,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:57,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:57,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:57,807 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:57,812 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:58,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:58,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:58,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:58,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:59,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:00,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:02,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:02,558 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:02,559 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:02,564 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:02,617 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:04,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:05,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:09,525 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:28:09,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:09,580 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:09,580 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:09,586 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:09,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:10,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:11,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:13,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:13,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:13,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:15,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:15,302 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:15,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:15,308 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:15,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:16,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:17,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:21,060 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:21,061 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:21,061 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:21,067 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:21,645 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:22,648 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:23,068 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:28,068 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:28,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:28,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:28,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:30,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:30,031 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:30,031 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:30,036 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:30,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:32,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:33,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:35,096 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:35,097 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:35,098 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:35,103 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:35,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:36,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:38,104 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:39,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:39,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:39,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:39,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:39,527 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:28:39,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:40,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:43,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:43,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:43,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:43,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:44,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:44,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:44,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:44,500 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:44,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:46,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:49,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:50,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:50,114 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:50,114 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:50,119 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:50,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:52,692 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:54,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:54,572 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:54,572 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:54,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:54,578 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:54,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:56,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:58,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:58,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:58,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:59,575 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:59,576 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:59,576 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:59,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:59,581 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:59,701 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:00,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:03,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:03,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:03,482 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:03,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:03,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:04,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:05,488 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:08,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:08,624 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:08,624 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:08,630 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:08,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:09,530 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:29:10,531 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:10,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:13,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:13,142 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:13,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:13,147 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:13,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:13,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:13,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:13,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:14,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:15,759 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:17,804 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:17,804 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:17,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:17,810 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:18,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:20,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:20,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:24,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:24,620 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:24,621 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:24,627 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:24,738 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:26,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:26,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:28,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:28,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:28,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:30,067 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:30,067 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:30,067 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:30,074 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:30,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:32,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:32,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:34,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:34,963 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:34,963 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:34,969 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:35,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:36,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:37,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:39,535 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:29:40,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:40,341 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:40,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:40,347 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:40,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:42,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:43,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:43,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:43,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:43,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:46,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:46,694 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:46,694 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:46,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:46,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:48,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:48,775 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:51,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:51,986 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:51,986 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:51,992 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:52,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:53,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:54,784 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:58,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:58,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:58,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:58,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:58,680 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:58,680 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:58,686 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:58,787 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:59,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:00,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:04,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:07,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:07,126 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:07,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:07,131 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:07,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:08,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:09,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:30:10,535 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:11,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:11,657 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:11,657 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:11,662 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:11,808 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:12,811 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:13,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:13,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:13,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:15,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:15,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:15,977 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:15,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:15,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:16,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:18,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:21,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:23,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:23,554 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:23,554 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:23,559 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:23,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:24,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:26,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:28,225 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:28,226 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:28,226 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:28,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:28,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:28,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:28,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:28,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:30,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:31,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:32,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:32,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:32,894 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:32,899 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:33,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:34,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:36,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:37,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:37,628 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:37,629 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:37,634 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:37,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:38,849 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:39,539 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:30:41,755 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:41,756 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:41,756 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:41,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:41,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:42,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:42,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:43,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:43,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:43,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:46,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:46,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:46,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:46,559 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:46,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:48,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:48,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:50,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:50,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:50,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:50,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:51,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:52,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:53,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:56,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:56,900 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:56,900 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:56,905 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:57,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:58,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:58,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:58,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:58,879 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:59,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:03,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:03,419 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:03,419 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:03,425 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:03,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:04,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:05,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:08,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:08,390 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:08,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:08,396 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:08,892 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:09,538 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:31:10,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:10,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:13,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:13,234 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:13,234 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:13,240 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:13,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:13,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:13,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:13,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:14,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:15,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:19,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:19,330 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:19,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:19,337 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:19,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:20,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:21,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:23,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:23,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:23,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:23,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:23,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:24,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:26,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:28,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:28,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:28,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:28,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:28,795 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:28,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:28,801 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:28,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:30,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:31,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:32,826 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:32,826 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:32,826 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:32,832 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:32,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:34,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:37,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:39,541 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:31:41,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:41,321 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:41,321 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:41,327 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:41,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:42,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:43,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:43,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:43,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:46,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:46,018 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:46,018 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:46,024 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:46,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:48,954 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:49,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:50,279 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:50,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:50,280 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:50,286 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:50,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:52,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:54,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:55,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:55,376 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:55,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:55,381 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:55,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:56,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:58,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:58,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:58,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:59,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:59,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:59,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:59,954 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:59,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:59,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:02,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:04,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:04,261 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:04,261 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:04,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:04,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:05,268 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:06,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:09,543 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:32:10,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:10,285 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:10,285 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:10,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:10,291 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:10,985 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:12,988 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:13,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:13,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:13,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:15,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:15,354 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:15,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:15,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:15,361 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:15,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:16,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:20,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:22,507 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:22,508 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:22,508 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:22,514 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:23,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:25,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:25,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:27,024 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:27,025 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:27,025 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:27,031 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:28,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:28,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:28,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:28,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:29,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:30,686 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:32,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:32,970 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:32,970 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:32,976 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:33,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:35,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:36,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:37,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:37,511 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:37,511 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:37,517 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:38,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:39,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:39,545 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:32:41,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:41,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:41,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:41,926 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:41,932 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:42,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:43,035 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:43,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:43,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:43,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:46,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:46,319 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:46,319 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:46,325 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:47,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:47,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:49,042 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:51,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:51,641 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:51,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:51,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:52,047 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:52,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:53,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:57,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:58,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:58,063 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:58,063 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:58,068 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:58,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:58,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:58,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:59,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:01,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:03,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:03,063 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:03,064 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:03,064 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:03,069 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:04,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:05,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:08,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:09,547 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:33:10,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:10,039 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:10,039 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:10,045 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:10,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:13,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:13,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:13,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:13,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:13,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:15,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:15,850 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:15,850 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:15,856 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:16,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:17,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:18,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:21,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:21,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:21,965 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:21,970 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:22,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:23,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:23,971 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:26,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:26,670 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:26,670 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:26,676 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:27,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:28,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:28,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:28,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:29,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:29,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:31,095 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:31,096 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:31,096 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:31,102 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:31,105 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:33,109 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:35,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:35,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:35,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:35,318 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:35,323 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:36,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:37,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:39,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:33:40,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:43,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:43,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:43,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:43,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:43,703 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:43,703 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:43,709 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:44,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:45,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:45,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:47,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:47,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:47,883 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:47,888 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:48,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:49,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:50,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:52,808 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:52,809 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:52,809 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:52,815 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:53,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:55,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:56,815 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:58,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:58,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:58,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:59,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:59,198 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:59,198 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:59,204 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:00,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:01,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:02,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:05,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:05,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:05,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:05,292 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:06,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:07,159 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:07,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:09,555 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:34:12,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:12,156 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:12,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:12,161 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:12,165 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:13,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:13,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:13,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:15,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:17,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:17,731 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:17,731 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:17,737 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:18,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:18,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:19,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:23,774 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:25,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:25,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:25,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:25,733 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:26,187 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:27,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:28,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:28,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:28,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:29,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:30,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:30,409 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:30,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:30,416 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:31,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:33,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:35,247 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:35,248 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:35,248 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:35,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:35,254 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:36,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:37,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:39,554 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:34:40,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:40,108 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:40,108 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:40,113 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:40,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:41,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:43,213 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:43,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:43,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:43,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:46,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:48,575 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:48,576 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:48,576 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:48,582 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:49,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:51,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:52,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:52,782 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:52,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:52,783 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:52,788 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:53,225 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:55,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:57,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:57,225 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:57,225 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:57,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:57,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:58,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:58,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:58,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:58,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:59,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:02,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:02,937 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:02,938 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:02,943 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:03,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:03,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:05,245 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:07,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:07,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:07,894 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:07,899 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:08,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:09,251 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:09,559 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:35:09,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:12,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:12,459 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:12,459 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:12,465 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:13,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:13,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:13,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:13,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:14,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:15,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:17,980 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:17,981 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:17,981 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:17,986 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:18,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:19,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:20,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:22,503 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:22,504 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:22,505 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:22,510 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:23,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:25,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:25,511 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:27,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:27,066 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:27,066 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:27,072 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:27,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:28,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:28,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:28,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:29,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:30,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:31,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:31,459 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:31,459 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:31,465 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:32,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:33,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:35,950 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:35,951 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:35,951 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:35,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:35,956 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:36,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:37,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:39,558 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:35:40,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:40,442 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:40,442 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:40,448 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:41,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:41,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:43,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:43,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:43,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:43,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:46,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:48,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:48,591 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:48,591 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:48,597 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:49,308 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:51,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:51,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:52,916 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:52,917 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:52,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:52,923 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:53,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:55,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:56,924 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:57,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:57,970 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:57,970 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:57,975 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:58,323 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:58,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:58,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:58,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:59,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:02,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:02,718 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:02,718 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:02,718 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:02,723 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:03,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:05,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:07,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:07,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:07,203 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:07,208 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:07,336 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:08,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:09,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:09,564 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:36:12,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:12,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:12,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:12,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:12,344 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:13,216 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:13,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:13,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:13,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:15,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:16,944 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:16,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:16,945 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:16,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:17,359 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:18,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:19,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:22,708 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:22,709 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:22,709 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:22,716 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:23,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:24,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:25,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:28,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:28,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:28,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:30,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:31,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:31,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:31,044 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:31,050 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:31,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:33,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:36,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:37,795 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:37,796 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:37,796 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:37,801 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:38,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:39,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:39,562 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:36:41,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:42,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:42,624 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:42,624 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:42,630 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:43,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:43,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:43,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:43,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:45,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:46,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:47,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:47,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:47,033 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:47,039 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:47,396 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:49,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:52,074 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:54,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:54,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:54,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:54,216 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:54,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:55,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:57,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:58,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:58,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:58,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:59,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:59,499 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:59,499 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:59,504 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:00,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:01,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:02,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:04,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:04,261 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:04,261 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:04,266 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:04,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:07,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:08,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:09,564 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:37:12,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:12,609 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:12,609 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:12,615 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:13,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:13,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:13,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:13,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:13,696 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:15,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:16,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:16,677 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:16,677 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:16,683 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:17,438 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:18,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:19,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:22,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:22,607 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:22,608 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:22,613 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:23,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:24,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:25,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:28,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:28,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:28,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:29,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:30,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:30,923 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:30,923 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:30,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:31,458 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:33,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:34,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:36,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:36,701 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:36,701 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:36,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:37,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:39,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:39,570 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:37:40,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:41,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:41,170 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:41,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:41,176 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:41,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:43,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:43,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:43,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:43,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:45,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:46,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:46,290 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:46,290 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:46,296 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:46,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:49,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:51,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:53,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:53,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:53,954 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:53,960 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:54,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:55,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:56,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:58,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:58,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:58,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:02,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:02,240 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:02,240 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:02,240 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:02,246 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:02,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:03,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:07,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:07,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:07,998 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:07,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:08,004 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:08,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:09,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:09,569 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:38:12,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:13,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:13,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:13,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:13,162 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:13,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:13,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:13,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:13,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:15,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:17,606 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:20,767 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:20,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:20,768 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:20,774 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:21,530 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:22,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:23,536 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:24,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:24,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:24,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:24,934 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:25,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:27,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:27,935 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:28,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:28,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:28,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:29,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:29,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:29,024 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:29,030 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:29,545 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:31,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:33,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:35,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:35,188 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:35,188 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:35,194 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:35,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:37,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:38,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:39,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:38:41,707 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:41,708 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:41,708 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:41,713 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:42,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:43,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:43,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:43,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:43,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:43,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:48,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:48,457 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:48,457 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:48,463 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:48,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:49,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:51,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:53,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:53,502 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:53,503 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:53,508 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:53,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:54,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:55,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:58,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:58,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:58,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:59,692 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:01,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:01,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:01,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:01,923 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:02,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:03,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:04,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:09,573 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:39:10,575 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:10,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:10,609 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:10,609 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:10,615 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:11,603 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:13,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:13,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:13,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:13,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:15,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:16,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:16,329 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:16,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:16,335 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:16,612 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:17,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:21,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:21,215 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:21,215 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:21,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:21,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:21,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:23,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:26,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:26,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:26,779 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:26,779 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:26,785 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:27,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:28,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:28,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:28,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:29,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:31,604 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:31,788 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:31,788 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:31,788 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:31,794 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:32,636 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:33,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:36,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:38,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:38,082 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:38,082 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:38,087 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:38,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:39,575 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:39:39,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:42,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:42,311 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:42,311 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:42,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:42,317 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:42,650 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:43,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:43,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:43,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:43,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:47,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:50,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:50,838 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:50,838 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:50,844 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:51,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:52,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:53,666 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:56,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:56,786 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:56,786 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:56,791 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:57,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:58,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:58,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:58,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:58,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:59,675 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:01,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:01,660 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:01,660 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:01,666 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:01,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:03,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:03,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:05,745 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:05,746 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:05,746 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:05,751 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:06,684 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:07,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:08,752 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:09,578 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:40:10,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:10,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:10,030 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:10,036 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:10,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:11,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:13,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:13,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:13,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:14,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:16,844 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:16,844 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:16,845 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:16,850 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:17,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:19,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:19,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:23,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:23,594 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:23,594 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:23,600 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:23,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:25,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:25,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:28,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:28,356 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:28,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:28,362 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:28,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:28,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:28,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:28,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:29,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:30,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:35,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:35,456 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:35,456 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:35,462 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:35,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:36,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:37,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:39,580 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:40:41,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:41,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:41,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:41,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:41,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:42,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:43,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:43,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:43,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:43,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:47,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:48,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:48,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:48,323 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:48,328 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:48,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:49,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:53,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:53,591 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:53,592 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:53,592 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:53,598 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:53,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:55,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:57,831 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:57,832 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:57,832 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:57,838 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:58,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:58,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:58,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:58,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:58,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:59,763 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:03,692 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:06,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:06,311 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:06,311 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:06,317 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:06,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:07,775 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:09,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:09,585 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:41:12,911 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:12,912 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:12,912 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:12,918 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:13,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:13,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:13,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:13,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:14,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:15,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:18,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:18,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:18,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:18,457 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:18,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:19,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:20,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:23,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:23,722 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:23,723 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:23,728 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:23,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:25,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:25,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:28,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:28,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:28,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:28,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:28,687 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:28,687 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:28,693 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:28,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:31,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:31,811 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:33,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:33,641 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:33,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:33,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:33,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:35,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:37,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:38,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:38,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:38,013 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:38,019 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:38,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:39,584 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:41:39,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:43,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:43,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:43,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:43,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:45,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:45,275 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:45,276 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:45,281 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:45,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:47,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:49,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:50,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:50,782 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:50,782 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:50,788 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:50,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:53,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:54,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:55,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:55,568 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:55,568 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:55,574 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:55,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:57,849 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:58,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:58,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:58,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:00,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:00,269 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:00,270 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:00,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:00,275 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:00,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:01,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:05,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:05,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:05,770 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:05,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:05,776 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:05,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:07,863 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:09,590 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:42:10,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:10,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:10,678 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:10,679 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:10,685 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:10,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:13,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:13,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:13,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:13,872 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:15,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:16,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:16,381 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:16,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:16,387 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:16,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:17,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:21,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:22,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:22,080 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:22,080 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:22,086 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:22,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:23,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:27,087 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:27,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:27,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:27,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:27,170 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:27,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:28,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:28,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:28,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:29,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:31,900 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:31,901 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:31,901 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:31,907 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:32,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:32,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:33,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:37,942 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:38,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:38,649 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:38,649 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:38,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:38,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:39,589 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:42:41,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:43,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:43,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:43,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:43,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:43,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:43,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:43,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:43,733 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:43,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:45,919 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:48,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:48,403 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:48,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:48,409 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:48,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:49,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:49,925 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:52,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:52,821 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:52,821 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:52,826 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:52,928 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:54,827 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:55,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:58,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:58,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:58,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:58,544 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:58,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:58,608 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:58,613 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:58,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:59,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:00,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:03,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:03,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:03,307 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:03,313 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:03,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:05,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:06,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:08,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:08,397 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:08,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:08,403 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:08,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:09,597 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:43:09,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:11,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:13,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:13,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:13,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:13,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:13,648 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:13,649 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:13,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:13,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:14,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:16,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:19,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:19,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:19,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:19,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:19,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:20,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:22,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:25,055 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:25,056 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:25,056 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:25,062 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:25,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:26,984 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:28,062 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:28,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:28,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:28,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:29,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:29,968 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:29,968 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:29,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:29,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:30,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:34,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:36,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:36,124 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:36,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:36,130 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:36,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:36,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:39,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:39,593 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:43:40,604 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:40,605 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:40,605 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:40,611 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:41,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:41,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:43,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:43,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:43,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:44,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:45,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:45,161 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:45,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:45,166 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:46,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:47,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:49,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:49,671 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:49,671 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:49,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:49,677 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:50,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:51,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:54,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:56,898 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:56,899 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:56,899 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:56,905 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:57,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:58,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:58,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:58,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:59,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:00,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:02,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:02,366 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:02,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:02,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:03,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:03,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:06,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:06,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:06,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:06,643 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:06,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:07,061 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:07,061 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:09,595 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:44:11,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:11,060 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:11,060 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:11,066 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:11,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:12,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:13,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:13,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:13,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:13,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:16,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:16,997 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:16,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:17,004 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:17,073 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:18,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:19,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:22,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:22,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:22,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:22,249 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:23,084 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:23,084 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:23,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:28,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:28,052 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:28,052 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:28,058 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:28,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:28,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:28,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:28,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:28,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:29,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:33,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:34,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:34,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:34,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:34,133 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:35,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:35,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:38,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:38,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:38,807 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:38,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:38,813 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:39,105 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:39,597 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:44:41,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:43,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:43,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:43,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:43,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:43,647 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:43,647 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:43,653 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:44,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:44,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:45,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:48,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:48,161 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:48,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:48,167 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:49,123 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:49,123 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:50,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:52,095 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:52,096 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:52,097 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:52,103 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:52,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:53,129 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:56,104 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:56,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:56,523 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:56,523 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:56,529 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:57,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:57,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:58,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:58,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:58,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:01,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:01,012 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:01,013 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:01,018 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:01,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:02,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:03,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:05,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:05,624 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:05,624 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:05,629 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:06,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:07,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:07,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:09,599 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:45:10,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:10,845 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:10,846 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:10,852 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:11,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:12,852 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:13,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:13,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:13,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:13,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:15,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:15,681 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:15,682 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:15,688 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:16,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:17,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:18,725 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:21,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:21,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:21,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:21,128 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:21,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:23,173 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:24,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:27,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:27,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:27,436 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:27,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:28,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:28,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:28,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:28,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:29,182 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:29,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:31,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:31,910 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:31,910 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:31,916 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:32,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:33,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:34,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:36,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:36,204 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:36,204 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:36,210 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:37,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:37,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:39,602 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:45:40,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:40,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:40,862 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:40,862 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:40,867 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:41,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:43,202 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:43,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:43,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:43,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:45,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:45,428 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:45,428 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:45,433 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:46,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:46,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:47,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:49,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:49,572 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:49,572 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:49,577 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:50,211 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:51,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:51,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:55,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:55,197 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:55,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:55,203 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:55,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:57,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:57,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:58,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:58,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:58,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:02,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:03,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:03,444 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:03,444 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:03,450 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:04,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:05,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:08,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:08,435 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:08,435 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:08,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:08,441 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:09,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:09,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:09,604 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:46:12,575 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:12,576 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:12,576 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:12,583 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:13,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:13,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:13,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:13,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:13,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:13,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:18,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:20,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:20,967 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:20,967 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:20,973 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:21,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:23,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:24,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:26,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:26,770 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:26,770 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:26,776 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:27,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:27,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:28,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:28,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:28,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:29,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:31,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:31,380 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:31,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:31,386 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:32,274 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:33,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:35,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:36,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:36,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:36,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:36,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:37,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:37,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:39,609 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:46:40,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:41,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:41,518 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:41,518 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:41,524 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:42,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:43,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:43,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:43,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:43,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:45,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:45,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:45,835 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:45,835 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:45,842 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:46,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:47,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:50,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:53,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:53,966 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:53,967 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:53,972 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:54,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:55,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:55,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:58,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:58,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:58,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:59,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:59,170 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:59,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:59,176 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:59,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:01,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:01,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:04,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:04,621 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:04,621 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:04,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:05,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:05,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:06,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:08,779 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:08,780 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:08,780 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:08,786 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:09,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:09,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:09,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:47:12,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:13,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:13,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:13,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:13,907 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:13,908 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:13,908 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:13,914 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:14,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:15,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:17,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:18,591 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:18,592 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:18,593 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:18,599 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:19,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:19,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:22,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:22,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:22,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:22,935 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:23,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:23,936 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:25,353 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:27,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:27,615 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:27,615 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:27,621 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:28,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:28,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:28,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:28,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:29,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:29,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:33,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:33,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:33,039 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:33,044 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:33,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:35,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:35,369 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:37,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:37,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:37,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:37,355 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:37,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:39,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:39,613 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:47:40,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:41,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:41,557 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:41,557 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:41,563 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:42,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:43,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:43,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:43,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:43,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:45,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:46,054 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:46,055 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:46,055 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:46,062 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:46,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:47,386 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:50,703 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:50,704 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:50,704 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:50,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:50,710 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:51,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:51,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:55,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:57,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:57,106 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:57,106 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:57,112 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:57,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:58,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:58,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:58,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:59,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:01,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:02,667 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:02,667 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:02,667 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:02,673 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:03,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:03,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:06,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:07,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:07,678 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:07,678 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:07,684 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:08,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:09,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:09,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:48:12,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:12,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:12,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:12,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:12,648 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:13,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:13,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:13,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:13,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:13,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:17,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:17,503 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:17,503 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:17,510 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:18,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:18,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:19,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:22,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:22,070 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:22,070 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:22,076 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:22,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:23,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:24,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:28,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:28,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:28,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:29,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:30,486 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:30,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:30,487 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:30,493 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:31,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:31,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:35,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:35,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:35,906 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:35,906 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:35,912 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:36,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:37,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:39,618 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:48:40,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:40,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:40,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:40,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:40,819 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:41,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:41,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:43,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:43,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:43,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:45,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:47,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:47,365 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:47,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:47,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:47,472 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:49,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:51,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:53,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:53,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:53,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:53,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:53,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:55,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:57,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:58,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:58,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:58,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:01,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:01,266 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:01,266 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:01,272 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:01,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:02,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:03,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:06,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:06,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:06,494 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:06,500 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:06,501 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:07,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:07,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:09,616 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:49:12,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:12,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:12,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:12,702 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:12,708 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:13,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:13,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:13,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:13,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:13,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:17,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:19,683 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:19,684 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:19,684 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:19,690 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:20,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:21,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:22,725 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:24,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:24,411 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:24,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:24,416 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:24,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:25,530 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:28,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:28,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:28,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:28,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:31,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:31,214 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:31,214 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:31,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:31,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:33,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:34,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:36,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:36,342 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:36,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:36,348 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:36,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:37,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:39,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:39,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:49:41,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:41,891 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:41,891 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:41,897 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:42,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:43,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:43,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:43,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:43,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:44,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:47,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:47,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:47,659 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:47,665 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:48,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:49,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:49,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:52,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:52,045 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:52,045 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:52,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:52,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:53,572 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:55,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:56,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:56,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:56,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:56,156 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:56,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:57,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:58,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:58,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:58,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:00,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:00,715 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:00,715 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:00,715 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:00,722 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:01,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:01,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:05,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:06,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:06,620 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:06,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:06,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:07,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:07,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:09,624 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:50:11,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:11,142 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:11,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:11,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:11,148 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:11,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:13,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:13,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:13,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:13,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:16,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:17,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:17,114 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:17,114 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:17,120 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:17,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:19,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:22,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:22,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:22,316 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:22,316 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:22,323 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:22,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:23,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:27,320 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:27,321 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:27,321 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:27,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:27,327 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:27,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:28,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:28,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:28,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:29,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:32,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:32,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:32,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:32,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:32,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:32,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:33,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:36,851 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:36,852 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:36,852 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:36,858 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:37,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:37,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:37,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:39,623 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:50:43,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:43,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:43,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:43,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:45,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:45,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:45,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:45,174 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:45,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:47,652 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:49,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:49,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:49,998 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:49,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:50,004 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:50,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:51,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:55,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:55,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:55,769 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:55,770 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:55,776 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:56,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:57,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:58,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:58,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:58,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:00,279 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:00,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:00,280 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:00,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:00,286 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:00,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:01,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:05,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:07,718 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:07,719 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:07,719 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:07,725 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:08,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:09,628 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:51:09,684 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:10,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:11,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:11,801 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:11,801 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:11,807 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:12,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:13,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:13,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:13,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:15,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:16,503 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:16,504 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:16,504 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:16,510 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:16,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:17,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:21,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:21,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:21,968 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:21,968 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:21,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:22,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:23,707 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:26,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:28,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:28,124 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:28,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:28,131 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:28,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:28,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:28,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:29,715 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:32,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:32,582 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:32,582 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:32,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:32,588 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:32,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:33,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:37,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:37,050 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:37,050 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:37,056 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:37,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:37,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:38,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:39,628 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:51:41,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:41,311 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:41,311 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:41,318 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:41,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:43,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:43,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:43,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:43,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:43,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:45,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:45,700 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:45,701 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:45,706 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:45,738 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:47,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:48,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:50,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:50,294 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:50,294 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:50,300 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:50,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:51,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:54,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:55,835 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:55,836 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:55,836 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:55,841 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:56,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:57,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:58,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:58,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:58,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:59,656 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:00,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:00,114 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:00,114 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:00,121 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:00,759 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:01,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:04,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:04,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:04,613 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:04,620 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:04,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:05,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:05,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:09,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:09,240 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:09,240 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:09,246 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:09,633 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:52:09,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:09,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:10,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:13,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:13,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:14,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:14,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:14,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:14,161 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:14,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:15,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:16,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:19,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:19,174 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:19,174 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:19,180 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:19,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:19,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:22,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:24,944 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:24,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:24,945 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:24,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:25,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:25,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:27,952 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:28,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:28,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:29,240 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:29,241 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:29,241 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:29,246 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:29,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:31,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:33,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:38,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:38,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:38,462 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:38,462 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:38,468 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:38,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:39,632 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:52:39,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:42,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:42,682 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:42,682 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:42,689 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:42,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:43,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:43,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:43,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:43,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:43,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:47,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:47,368 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:47,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:47,374 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:47,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:49,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:49,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:51,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:51,793 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:51,793 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:51,799 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:51,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:53,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:54,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:56,847 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:56,848 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:56,848 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:56,854 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:57,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:57,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:58,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:58,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:58,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:00,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:00,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:00,975 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:00,975 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:00,981 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:01,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:01,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:06,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:06,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:06,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:06,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:06,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:06,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:07,860 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:09,634 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:53:11,273 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:11,277 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:11,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:11,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:11,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:11,867 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:11,867 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:13,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:13,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:15,606 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:15,606 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:15,606 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:15,612 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:15,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:16,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:17,875 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:21,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:21,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:21,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:21,163 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:21,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:21,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:22,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:25,891 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:25,892 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:25,892 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:25,898 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:26,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:27,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:27,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:28,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:28,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:30,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:30,822 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:30,822 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:30,828 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:30,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:31,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:33,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:35,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:35,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:35,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:35,777 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:35,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:37,905 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:39,637 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:53:39,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:40,788 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:40,789 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:40,789 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:40,794 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:40,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:41,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:43,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:43,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:43,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:44,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:45,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:45,351 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:45,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:45,357 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:45,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:47,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:50,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:52,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:52,968 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:52,968 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:52,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:53,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:53,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:55,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:58,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:58,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:58,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:01,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:03,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:03,998 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:03,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:04,004 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:04,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:05,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:07,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:09,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:09,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:09,501 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:09,508 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:09,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:54:09,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:11,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:12,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:13,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:13,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:15,976 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:15,977 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:15,977 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:15,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:16,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:17,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:18,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:21,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:21,216 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:21,216 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:21,222 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:21,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:21,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:23,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:26,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:26,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:26,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:26,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:26,974 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:27,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:28,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:28,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:28,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:31,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:31,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:31,870 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:31,877 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:31,984 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:33,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:33,988 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:36,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:36,357 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:36,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:36,363 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:36,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:37,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:39,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:39,641 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:54:40,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:40,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:40,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:40,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:40,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:41,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:43,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:43,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:43,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:44,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:45,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:45,116 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:45,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:45,122 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:46,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:46,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:49,920 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:49,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:49,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:49,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:49,927 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:50,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:52,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:54,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:54,549 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:54,549 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:54,555 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:55,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:55,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:56,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:58,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:58,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:58,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:58,840 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:58,841 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:58,841 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:58,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:59,022 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:00,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:00,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:05,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:06,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:06,015 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:06,015 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:06,021 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:06,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:08,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:09,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:55:10,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:10,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:10,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:10,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:11,040 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:11,479 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:12,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:13,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:13,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:13,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:16,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:16,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:16,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:16,323 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:17,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:17,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:18,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:22,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:25,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:25,160 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:25,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:25,166 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:26,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:26,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:28,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:28,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:28,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:28,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:30,468 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:30,469 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:30,469 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:30,475 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:31,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:32,073 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:33,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:38,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:38,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:38,763 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:38,763 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:38,769 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:39,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:39,648 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:55:40,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:43,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:43,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:43,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:43,622 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:43,720 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:43,721 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:43,721 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:43,727 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:44,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:46,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:48,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:49,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:49,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:49,047 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:49,053 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:49,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:50,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:53,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:53,706 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:53,706 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:53,712 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:54,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:54,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:56,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:58,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:58,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:58,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:58,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:58,696 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:58,696 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:58,702 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:59,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:00,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:00,703 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:05,703 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:06,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:06,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:06,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:06,824 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:07,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:08,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:09,647 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:56:11,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:12,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:12,718 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:12,718 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:12,724 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:13,131 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:13,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:13,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:13,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:14,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:17,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:18,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:18,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:18,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:18,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:19,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:20,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:23,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:24,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:24,583 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:24,583 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:24,589 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:25,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:26,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:28,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:28,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:28,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:28,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:29,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:29,698 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:29,698 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:29,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:30,159 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:32,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:33,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:34,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:34,356 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:34,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:34,363 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:35,165 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:36,169 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:39,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:39,652 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:56:40,068 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:40,069 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:40,069 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:40,075 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:40,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:42,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:43,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:43,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:43,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:44,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:44,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:44,474 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:44,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:44,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:45,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:46,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:49,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:49,136 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:49,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:49,142 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:49,185 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:50,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:50,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:54,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:54,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:54,212 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:54,218 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:55,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:55,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:56,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:58,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:58,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:58,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:58,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:58,615 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:58,615 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:58,621 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:59,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:00,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:00,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:04,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:04,188 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:04,188 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:04,194 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:04,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:06,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:06,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:09,652 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:57:09,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:09,782 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:09,782 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:09,788 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:10,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:11,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:12,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:13,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:13,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:14,273 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:14,274 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:14,274 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:14,280 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:15,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:16,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:17,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:20,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:20,200 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:20,200 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:20,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:20,231 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:22,237 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:23,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:24,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:24,641 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:24,641 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:24,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:25,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:26,242 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:28,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:28,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:28,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:33,544 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:33,544 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:33,545 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:33,550 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:34,254 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:34,254 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:34,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:38,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:38,626 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:38,627 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:38,632 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:39,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:39,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:39,655 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:57:40,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:43,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:43,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:43,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:45,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:46,828 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:46,829 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:46,829 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:46,835 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:47,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:48,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:50,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:51,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:51,384 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:51,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:51,389 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:52,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:52,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:56,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:56,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:56,510 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:56,510 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:56,516 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:57,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:58,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:58,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:58,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:58,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:01,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:01,197 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:01,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:01,203 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:01,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:02,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:02,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:07,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:07,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:07,538 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:07,538 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:07,544 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:08,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:08,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:09,657 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:58:12,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:12,175 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:12,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:12,181 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:12,308 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:13,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:13,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:13,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:14,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:18,703 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:19,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:19,346 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:19,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:19,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:20,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:20,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:24,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:24,900 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:24,902 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:24,902 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:24,907 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:25,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:26,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:28,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:28,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:29,707 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:29,752 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:29,752 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:29,752 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:29,758 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:30,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:32,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:34,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:34,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:34,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:34,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:35,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:35,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:36,346 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:39,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:39,352 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:39,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:39,358 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:39,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:58:40,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:40,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:40,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:43,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:43,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:43,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:44,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:44,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:44,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:44,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:44,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:46,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:46,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:49,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:49,459 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:49,459 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:49,465 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:50,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:50,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:51,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:53,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:53,588 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:53,588 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:53,594 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:54,372 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:54,373 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:56,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:58,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:58,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:58,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:01,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:02,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:02,646 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:02,646 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:02,653 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:03,382 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:04,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:06,690 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:07,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:07,050 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:07,050 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:07,056 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:07,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:08,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:09,665 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:59:11,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:11,742 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:11,742 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:11,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:11,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:12,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:13,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:13,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:14,402 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:17,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:17,830 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:17,830 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:17,830 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:17,836 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:18,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:20,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:22,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:22,460 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:22,460 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:22,466 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:23,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:23,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:24,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:27,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:27,321 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:27,322 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:27,328 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:27,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:28,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:28,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:28,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:28,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:31,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:31,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:31,664 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:31,670 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:32,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:32,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:33,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:36,656 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:36,657 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:36,657 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:36,664 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:37,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:38,438 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:39,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:59:39,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:42,544 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:42,545 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:42,545 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:42,551 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:43,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:43,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:43,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:43,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:44,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:45,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:49,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:49,594 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:49,594 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:49,600 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:50,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:50,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:50,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:55,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:58,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:58,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:58,030 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:58,036 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:58,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:58,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:58,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:58,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:00,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:00,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:02,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:02,254 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:02,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:02,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:02,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:04,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:06,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:07,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:07,081 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:07,081 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:07,087 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:07,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:08,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:09,669 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:00:11,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:11,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:11,838 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:11,838 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:11,844 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:12,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:13,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:13,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:14,491 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:17,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:17,560 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:17,561 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:17,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:17,566 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:18,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:18,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:22,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:22,946 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:22,947 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:22,947 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:22,953 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:23,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:24,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:27,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:28,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:28,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:31,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:31,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:31,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:31,355 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:31,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:32,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:33,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:36,031 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:36,032 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:36,032 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:36,038 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:36,521 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:38,526 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:39,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:39,668 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:00:40,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:40,560 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:40,560 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:40,565 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:41,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:42,533 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:43,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:43,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:43,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:44,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:49,026 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:49,027 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:49,027 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:49,033 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:49,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:50,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:50,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:54,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:54,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:54,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:54,924 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:55,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:55,924 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:56,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:58,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:58,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:58,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:00,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:00,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:00,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:00,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:00,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:01,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:02,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:06,446 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:06,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:06,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:06,498 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:06,503 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:06,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:08,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:09,673 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:01:11,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:11,587 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:11,588 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:11,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:11,593 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:12,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:12,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:13,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:13,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:13,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:16,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:16,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:16,996 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:16,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:17,003 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:17,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:18,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:22,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:26,118 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:26,119 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:26,119 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:26,126 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:26,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:27,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:28,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:28,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:28,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:31,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:31,357 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:31,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:31,364 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:31,603 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:32,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:32,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:36,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:36,423 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:36,424 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:36,430 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:36,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:37,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:38,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:39,672 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:01:42,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:42,276 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:42,276 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:42,282 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:42,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:43,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:43,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:43,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:43,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:44,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:46,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:46,800 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:46,800 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:46,807 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:47,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:48,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:48,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:51,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:51,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:51,044 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:51,051 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:51,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:52,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:54,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:57,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:57,560 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:57,560 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:57,566 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:57,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:58,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:58,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:58,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:58,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:59,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:01,804 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:01,804 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:01,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:01,811 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:02,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:02,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:04,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:08,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:08,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:08,659 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:08,665 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:09,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:09,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:02:10,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:10,676 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:13,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:13,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:13,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:14,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:14,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:14,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:14,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:14,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:16,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:16,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:19,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:19,435 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:19,435 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:19,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:19,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:20,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:22,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:25,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:25,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:25,515 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:25,521 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:25,680 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:26,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:27,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:28,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:28,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:30,760 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:30,761 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:30,761 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:30,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:31,692 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:32,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:32,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:36,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:36,040 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:36,040 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:36,046 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:36,699 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:38,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:38,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:39,680 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:02:40,562 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:40,563 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:40,563 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:40,569 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:40,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:42,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:43,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:43,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:43,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:43,667 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:46,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:46,676 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:46,676 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:46,682 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:46,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:48,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:48,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:51,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:51,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:51,821 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:51,827 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:52,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:52,724 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:53,827 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:58,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:58,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:58,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:58,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:58,698 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:58,699 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:58,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:58,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:59,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:00,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:03,319 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:03,320 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:03,321 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:03,327 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:03,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:04,741 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:05,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:08,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:08,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:08,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:08,174 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:08,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:09,679 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:03:10,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:10,747 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:12,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:12,589 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:12,589 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:12,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:12,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:13,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:13,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:13,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:14,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:16,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:17,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:17,060 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:17,060 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:17,066 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:17,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:18,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:21,712 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:21,712 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:21,713 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:21,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:21,718 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:21,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:22,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:26,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:26,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:26,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:26,313 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:26,771 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:27,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:28,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:28,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:28,778 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:30,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:30,557 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:30,558 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:30,564 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:30,780 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:32,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:32,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:35,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:35,617 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:35,618 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:35,624 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:35,784 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:36,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:37,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:39,685 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:03:41,915 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:41,916 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:41,917 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:41,923 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:42,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:42,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:42,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:43,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:43,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:43,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:48,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:49,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:49,282 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:49,282 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:49,288 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:49,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:50,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:53,912 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:53,913 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:53,913 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:53,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:53,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:54,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:54,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:58,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:58,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:58,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:58,808 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:58,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:58,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:58,815 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:58,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:59,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:00,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:04,850 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:05,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:05,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:05,033 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:05,039 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:05,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:06,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:09,683 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:04:09,723 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:09,724 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:09,724 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:09,731 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:09,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:10,732 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:10,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:13,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:13,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:13,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:16,639 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:19,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:19,892 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:19,892 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:19,899 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:20,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:20,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:21,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:24,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:24,488 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:24,488 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:24,494 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:24,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:26,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:27,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:28,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:28,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:29,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:29,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:29,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:29,950 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:30,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:30,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:32,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:34,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:34,450 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:34,450 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:34,456 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:34,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:36,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:38,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:39,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:39,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:39,239 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:39,244 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:39,689 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:04:39,879 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:40,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:43,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:43,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:43,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:43,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:44,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:44,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:44,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:44,060 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:44,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:46,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:48,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:48,952 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:48,952 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:48,952 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:48,958 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:49,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:50,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:53,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:53,558 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:53,558 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:53,564 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:53,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:54,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:54,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:58,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:58,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:58,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:59,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:00,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:00,277 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:00,277 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:00,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:00,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:02,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:04,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:04,634 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:04,634 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:04,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:04,640 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:04,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:06,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:09,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:09,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:09,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:09,156 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:09,688 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:05:09,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:09,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:10,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:13,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:13,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:13,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:13,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:13,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:13,659 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:13,665 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:13,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:14,932 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:15,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:18,323 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:18,324 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:18,324 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:18,330 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:18,936 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:20,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:21,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:22,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:22,879 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:22,879 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:22,885 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:22,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:24,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:26,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:27,591 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:27,592 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:27,592 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:27,599 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:27,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:28,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:28,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:28,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:32,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:32,151 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:32,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:32,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:32,157 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:32,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:34,962 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:37,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:37,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:37,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:37,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:37,161 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:37,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:38,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:39,694 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:05:41,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:41,685 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:41,685 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:41,691 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:41,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:42,692 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:42,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:43,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:43,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:43,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:46,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:46,193 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:46,193 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:46,199 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:46,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:48,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:48,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:53,095 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:53,096 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:53,096 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:53,103 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:53,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:54,103 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:54,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:58,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:58,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:58,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:58,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:58,709 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:58,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:58,715 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:58,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:59,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:00,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:03,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:03,835 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:03,835 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:03,841 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:04,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:04,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:05,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:09,692 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:06:10,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:11,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:11,037 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:11,037 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:11,043 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:12,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:13,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:13,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:13,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:13,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:15,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:17,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:17,421 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:17,421 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:17,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:18,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:19,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:21,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:22,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:22,459 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:22,459 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:22,465 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:23,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:25,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:26,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:27,563 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:27,564 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:27,564 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:27,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:28,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:28,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:28,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:29,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:31,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:31,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:31,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:31,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:31,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:32,047 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:33,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:36,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:37,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:37,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:37,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:37,356 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:38,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:39,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:39,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:06:42,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:42,003 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:42,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:42,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:42,009 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:42,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:43,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:43,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:43,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:43,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:46,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:46,376 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:46,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:46,383 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:47,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:47,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:49,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:52,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:54,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:54,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:54,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:54,890 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:55,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:57,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:57,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:58,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:58,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:58,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:00,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:00,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:00,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:00,321 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:01,086 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:03,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:03,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:04,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:04,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:04,743 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:04,749 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:05,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:07,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:08,750 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:09,697 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:07:10,443 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:10,443 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:10,444 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:10,450 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:11,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:13,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:13,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:13,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:13,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:14,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:17,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:17,191 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:17,191 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:17,197 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:18,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:19,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:20,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:21,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:21,663 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:21,663 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:21,669 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:22,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:23,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:25,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:28,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:28,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:28,212 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:28,218 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:28,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:28,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:29,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:29,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:30,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:34,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:34,302 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:34,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:34,308 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:35,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:36,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:37,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:39,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:39,616 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:39,617 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:39,622 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:39,699 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:07:40,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:41,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:41,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:43,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:43,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:43,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:46,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:46,052 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:46,052 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:46,057 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:46,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:47,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:47,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:51,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:51,166 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:51,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:51,171 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:52,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:52,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:53,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:57,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:57,736 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:57,737 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:57,737 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:57,743 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:58,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:58,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:58,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:58,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:59,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:02,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:02,413 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:02,413 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:02,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:02,419 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:03,178 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:05,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:06,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:06,541 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:06,541 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:06,547 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:07,185 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:07,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:09,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:09,704 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:08:12,706 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:13,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:13,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:13,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:14,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:14,794 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:14,794 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:14,800 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:15,196 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:17,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:17,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:18,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:18,922 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:18,923 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:18,928 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:19,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:21,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:22,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:24,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:24,019 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:24,019 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:24,025 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:24,210 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:25,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:28,026 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:28,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:28,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:28,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:28,684 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:28,684 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:28,690 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:29,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:31,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:33,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:33,418 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:33,418 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:33,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:33,424 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:34,225 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:35,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:38,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:38,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:38,663 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:38,663 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:38,669 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:39,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:39,703 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:08:41,237 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:43,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:43,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:43,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:43,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:47,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:47,303 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:47,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:47,310 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:48,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:49,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:49,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:53,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:53,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:53,926 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:53,932 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:54,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:54,932 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:55,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:58,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:58,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:58,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:59,667 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:59,667 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:59,667 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:59,673 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:00,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:00,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:01,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:04,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:04,326 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:04,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:04,332 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:05,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:05,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:06,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:09,708 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:09:09,822 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:09,822 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:09,822 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:09,828 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:10,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:11,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:11,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:13,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:13,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:13,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:17,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:18,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:18,362 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:18,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:18,368 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:19,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:21,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:23,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:23,715 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:23,716 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:23,716 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:23,722 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:24,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:25,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:28,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:28,282 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:28,282 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:28,288 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:28,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:28,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:28,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:28,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:29,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:33,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:33,768 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:33,769 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:33,769 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:33,775 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:34,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:35,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:38,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:39,708 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:09:42,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:42,344 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:42,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:42,350 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:43,328 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:43,328 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:43,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:43,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:43,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:44,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:49,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:49,275 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:49,275 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:49,280 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:49,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:50,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:51,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:53,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:53,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:53,608 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:53,613 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:54,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:55,346 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:55,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:58,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:58,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:58,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:58,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:58,644 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:58,644 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:58,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:59,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:00,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:01,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:04,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:04,956 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:04,956 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:04,962 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:05,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:05,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:07,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:09,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:09,435 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:09,435 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:09,441 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:09,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:10:10,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:11,369 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:11,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:13,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:13,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:13,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:14,084 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:14,085 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:14,085 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:14,090 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:14,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:15,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:17,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:18,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:18,653 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:18,653 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:18,658 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:19,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:21,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:22,659 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:23,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:23,413 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:23,414 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:23,419 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:24,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:25,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:27,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:27,873 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:27,874 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:27,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:27,879 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:28,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:28,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:28,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:28,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:29,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:32,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:32,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:32,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:32,346 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:32,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:33,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:33,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:36,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:36,742 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:36,742 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:36,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:37,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:38,748 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:39,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:39,712 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:10:41,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:41,050 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:41,050 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:41,055 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:41,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:43,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:43,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:43,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:43,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:44,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:46,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:46,001 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:46,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:46,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:46,420 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:47,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:50,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:51,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:51,384 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:51,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:51,390 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:51,427 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:53,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:55,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:57,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:57,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:57,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:57,580 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:58,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:58,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:58,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:58,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:59,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:00,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:02,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:02,366 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:02,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:02,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:02,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:03,449 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:06,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:07,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:07,146 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:07,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:07,152 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:07,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:09,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:09,714 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:11:11,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:12,232 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:12,233 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:12,233 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:12,239 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:12,458 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:13,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:13,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:13,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:13,462 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:17,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:18,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:18,712 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:18,712 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:18,718 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:19,469 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:21,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:22,754 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:24,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:24,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:24,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:24,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:24,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:25,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:28,320 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:28,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:28,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:28,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:29,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:29,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:29,202 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:29,207 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:29,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:31,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:34,246 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:34,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:34,296 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:34,296 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:34,302 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:34,491 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:35,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:39,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:39,720 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:11:42,919 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:42,920 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:42,920 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:42,926 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:43,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:43,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:43,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:43,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:44,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:45,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:49,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:49,822 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:49,823 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:49,823 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:49,830 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:50,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:51,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:54,074 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:54,075 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:54,075 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:54,081 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:54,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:55,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:55,521 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:58,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:58,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:58,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:00,712 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:00,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:00,928 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:00,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:00,934 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:01,530 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:03,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:05,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:06,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:06,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:06,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:06,292 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:06,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:07,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:09,719 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:12:10,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:10,834 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:10,835 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:10,840 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:11,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:11,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:13,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:13,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:13,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:13,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:17,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:20,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:20,738 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:20,738 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:20,744 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:21,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:22,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:23,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:26,894 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:26,895 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:26,895 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:26,900 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:27,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:27,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:28,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:28,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:28,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:29,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:32,180 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:32,181 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:32,181 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:32,187 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:32,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:33,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:33,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:38,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:39,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:39,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:39,289 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:39,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:39,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:39,724 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:12:41,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:43,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:43,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:43,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:43,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:44,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:44,132 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:44,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:44,138 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:44,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:45,597 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:48,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:48,858 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:48,859 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:48,859 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:48,864 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:49,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:51,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:53,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:55,454 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:55,455 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:55,455 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:55,461 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:55,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:57,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:58,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:58,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:58,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:59,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:00,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:00,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:00,897 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:00,903 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:01,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:03,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:04,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:05,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:05,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:05,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:05,573 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:05,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:07,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:09,723 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:13:10,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:10,266 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:10,266 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:10,266 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:10,272 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:10,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:11,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:13,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:13,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:13,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:15,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:15,179 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:15,180 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:15,185 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:15,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:16,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:17,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:20,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:20,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:20,945 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:20,950 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:21,648 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:21,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:23,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:26,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:27,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:27,290 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:27,290 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:27,296 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:27,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:28,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:28,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:28,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:29,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:31,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:31,555 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:31,555 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:31,561 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:31,666 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:32,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:33,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:36,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:36,066 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:36,067 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:36,072 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:36,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:37,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:38,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:39,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:13:41,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:41,303 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:41,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:41,309 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:41,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:43,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:43,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:43,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:43,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:43,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:45,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:45,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:45,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:45,733 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:46,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:47,690 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:48,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:50,231 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:50,231 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:50,231 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:50,238 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:50,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:51,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:54,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:54,983 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:54,983 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:54,984 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:54,990 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:55,699 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:57,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:58,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:58,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:58,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:59,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:59,399 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:59,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:59,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:59,406 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:59,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:01,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:03,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:03,810 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:03,810 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:03,816 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:04,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:04,816 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:05,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:09,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:14:10,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:10,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:10,465 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:10,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:10,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:10,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:11,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:13,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:13,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:13,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:15,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:18,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:18,855 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:18,855 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:18,861 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:19,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:20,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:21,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:24,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:24,583 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:24,583 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:24,589 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:24,743 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:25,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:26,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:28,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:28,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:28,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:29,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:29,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:29,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:29,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:29,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:31,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:32,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:35,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:35,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:35,865 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:35,870 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:36,760 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:37,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:37,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:39,730 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:14:40,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:40,758 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:40,758 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:40,764 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:40,766 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:41,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:43,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:43,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:43,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:43,714 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:47,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:47,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:47,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:47,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:47,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:49,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:49,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:53,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:53,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:53,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:53,170 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:53,786 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:54,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:55,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:58,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:58,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:58,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:58,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:58,693 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:58,694 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:58,699 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:58,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:59,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:59,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:02,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:02,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:02,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:02,984 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:03,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:04,984 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:05,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:07,468 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:07,469 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:07,469 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:07,475 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:07,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:09,732 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:15:09,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:10,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:13,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:13,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:13,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:13,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:13,653 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:13,653 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:13,659 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:13,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:15,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:16,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:19,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:19,599 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:19,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:19,605 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:19,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:21,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:22,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:26,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:26,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:26,323 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:26,328 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:26,834 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:27,838 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:28,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:28,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:28,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:28,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:30,772 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:30,773 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:30,773 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:30,779 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:30,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:31,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:33,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:35,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:35,204 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:35,204 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:35,211 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:35,849 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:37,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:39,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:39,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:39,419 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:39,419 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:39,426 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:39,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:15:39,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:41,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:43,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:43,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:43,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:44,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:44,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:44,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:44,654 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:44,660 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:44,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:45,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:49,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:49,339 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:49,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:49,344 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:49,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:50,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:51,875 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:53,875 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:53,875 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:53,876 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:53,881 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:54,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:55,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:55,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:58,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:58,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:58,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:59,443 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:59,444 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:59,444 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:59,450 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:59,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:01,485 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:01,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:03,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:03,685 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:03,686 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:03,691 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:03,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:05,896 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:06,692 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:08,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:08,001 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:08,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:08,008 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:08,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:09,739 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:16:09,902 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:11,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:12,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:12,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:12,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:12,573 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:12,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:13,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:16:13,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:16:13,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:16:13,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:17,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:19,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:19,401 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:19,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:19,406 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:19,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:21,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:23,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:24,532 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:24,533 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:24,533 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:24,538 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:24,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:25,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:28,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:16:28,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:16:28,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:16:28,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:29,119 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:29,119 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:29,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:29,125 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:29,932 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:31,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:33,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:33,570 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:33,570 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:33,576 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:33,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:34,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:35,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:39,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:39,101 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:39,101 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:39,108 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:39,739 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:16:39,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:39,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:41,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:43,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:16:43,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:16:43,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:16:44,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:44,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:44,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:44,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:44,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:45,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:45,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:48,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:48,967 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:48,968 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:48,973 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:49,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:50,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:51,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:55,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:55,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:55,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:55,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:55,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:56,480 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:57,974 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:58,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:16:58,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:16:58,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:17:00,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:00,330 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:00,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:00,336 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:00,976 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:01,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:02,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:07,076 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:07,077 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:07,077 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:07,083 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:07,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:08,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:09,744 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:17:09,992 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:11,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:11,722 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:11,722 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:11,729 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:11,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:13,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:17:13,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:17:13,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:17:13,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:13,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:16,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:16,214 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:16,214 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:16,219 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:17,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:18,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:19,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:24,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:24,811 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:24,812 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:24,812 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:24,818 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:25,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:25,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:28,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:17:28,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:17:28,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:17:29,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:30,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:30,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:30,058 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:30,064 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:31,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:31,024 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:35,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:35,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:35,632 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:35,632 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:35,638 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:36,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:37,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:39,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:17:40,745 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:41,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:41,601 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:41,601 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:41,607 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:42,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:43,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:43,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:17:43,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:17:43,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:17:46,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:46,140 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:46,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:46,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:46,145 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:47,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:47,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:51,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:51,147 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:51,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:51,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:51,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:52,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:53,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:56,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:56,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:56,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:56,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:56,869 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:57,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:57,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:58,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:17:58,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:17:58,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:01,690 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:01,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:01,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:01,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:01,870 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:02,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:03,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:06,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:06,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:06,950 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:06,950 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:06,956 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:07,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:07,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:09,748 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:18:12,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:12,230 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:12,230 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:12,230 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:12,236 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:13,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:13,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:13,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:18:13,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:18:13,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:17,672 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:18,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:18,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:18,675 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:18,680 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:19,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:19,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:22,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:23,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:23,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:23,930 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:23,935 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:24,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:25,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:27,936 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:28,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:18:28,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:18:28,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:28,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:28,956 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:28,956 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:28,962 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:29,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:29,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:32,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:34,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:34,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:34,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:34,446 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:35,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:35,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:38,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:39,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:39,717 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:39,717 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:39,723 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:39,748 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:18:40,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:41,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:43,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:18:43,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:18:43,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:43,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:44,768 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:44,769 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:44,769 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:44,775 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:45,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:45,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:48,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:49,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:49,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:49,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:49,100 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:49,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:51,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:53,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:53,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:53,964 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:53,964 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:53,970 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:54,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:55,147 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:58,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:58,367 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:58,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:58,374 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:58,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:18:58,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:18:58,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:59,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:59,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:59,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:02,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:02,277 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:02,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:02,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:03,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:03,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:05,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:09,750 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:19:10,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:10,552 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:10,552 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:10,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:10,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:11,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:11,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:13,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:19:13,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:19:13,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:19:15,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:15,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:15,658 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:15,658 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:15,665 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:16,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:17,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:20,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:21,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:21,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:21,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:21,303 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:22,185 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:23,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:25,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:25,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:25,872 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:25,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:25,878 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:26,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:27,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:28,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:19:28,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:19:28,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:19:30,745 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:30,746 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:30,746 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:30,752 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:31,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:31,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:31,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:36,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:38,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:38,676 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:38,676 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:38,682 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:39,211 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:39,211 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:39,752 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:19:42,754 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:43,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:19:43,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:19:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:19:44,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:44,236 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:44,236 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:44,241 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:45,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:45,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:48,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:49,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:49,442 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:49,442 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:49,448 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:50,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:51,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:53,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:53,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:53,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:53,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:53,984 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:54,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:55,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:58,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:58,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:58,322 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:58,328 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:58,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:19:58,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:19:58,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:19:58,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:59,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:59,242 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:03,070 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:03,071 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:03,071 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:03,077 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:03,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:03,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:04,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:07,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:07,386 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:07,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:07,392 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:08,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:09,256 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:09,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:09,755 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:20:11,798 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:11,798 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:11,798 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:11,804 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:12,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:13,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:13,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:20:13,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:20:13,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:20:14,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:17,086 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:17,087 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:17,087 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:17,093 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:17,269 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:17,269 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:20,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:22,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:22,029 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:22,029 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:22,036 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:22,274 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:23,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:26,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:27,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:27,361 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:27,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:27,367 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:28,283 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:28,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:20:28,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:20:28,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:20:29,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:31,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:32,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:32,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:32,515 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:32,521 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:33,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:33,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:37,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:37,289 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:37,290 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:37,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:37,296 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:37,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:39,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:39,760 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:20:42,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:42,161 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:42,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:42,168 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:42,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:43,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:43,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:43,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:20:43,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:20:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:20:46,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:46,938 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:46,938 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:46,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:47,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:47,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:48,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:53,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:53,930 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:53,930 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:53,936 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:54,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:54,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:55,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:58,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:58,256 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:58,256 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:58,262 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:58,328 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:58,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:20:58,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:20:58,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:20:59,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:00,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:03,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:03,193 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:03,193 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:03,199 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:03,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:05,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:06,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:08,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:08,148 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:08,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:08,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:08,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:09,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:09,759 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:21:11,761 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:13,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:13,295 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:13,296 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:13,301 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:13,349 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:13,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:21:13,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:21:13,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:21:15,355 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:17,699 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:19,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:19,323 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:19,323 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:19,329 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:19,359 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:21,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:23,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:24,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:24,308 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:24,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:24,314 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:24,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:25,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:28,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:21:28,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:21:28,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:21:28,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:32,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:32,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:32,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:32,740 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:33,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:33,382 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:33,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:38,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:38,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:38,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:38,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:39,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:39,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:39,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:39,764 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:21:43,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:21:43,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:21:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:21:44,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:44,325 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:44,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:44,331 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:44,396 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:45,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:45,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:48,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:48,822 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:48,822 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:48,828 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:49,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:49,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:50,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:53,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:53,902 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:53,902 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:53,908 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:54,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:55,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:55,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:58,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:21:58,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:21:58,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:21:59,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:59,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:59,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:59,293 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:59,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:01,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:01,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:06,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:07,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:07,711 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:07,711 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:07,717 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:08,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:09,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:09,763 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:22:11,765 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:13,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:22:13,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:22:13,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:22:14,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:14,879 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:14,879 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:14,885 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:15,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:15,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:16,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:19,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:19,359 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:19,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:19,365 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:19,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:21,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:22,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:23,987 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:23,988 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:23,988 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:23,994 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:24,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:25,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:27,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:28,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:22:28,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:22:28,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:22:29,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:29,091 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:29,091 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:29,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:29,465 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:29,465 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:33,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:35,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:35,958 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:35,958 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:35,964 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:36,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:37,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:38,964 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:39,768 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:22:40,731 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:40,732 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:40,732 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:40,738 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:41,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:41,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:43,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:22:43,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:22:43,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:22:44,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:44,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:44,970 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:44,970 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:44,976 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:45,488 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:45,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:50,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:55,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:57,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:57,365 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:57,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:57,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:57,503 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:58,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:22:58,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:22:58,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:22:59,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:00,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:03,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:03,070 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:03,070 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:03,076 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:03,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:03,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:06,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:07,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:07,561 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:07,561 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:07,567 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:08,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:09,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:09,768 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:23:11,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:12,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:12,428 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:12,428 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:12,434 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:12,526 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:13,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:23:13,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:23:13,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:23:13,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:17,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:18,476 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:18,477 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:18,477 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:18,483 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:18,536 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:19,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:22,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:22,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:22,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:22,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:22,860 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:23,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:23,545 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:26,891 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:26,891 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:26,892 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:26,898 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:27,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:27,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:27,898 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:28,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:23:28,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:23:28,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:23:33,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:33,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:33,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:33,871 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:33,876 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:34,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:35,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:38,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:38,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:38,934 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:38,934 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:38,940 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:39,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:39,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:39,770 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:23:43,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:23:43,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:23:43,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:23:43,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:43,794 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:43,794 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:43,800 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:44,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:44,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:45,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:49,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:50,219 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:50,220 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:50,220 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:50,226 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:50,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:51,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:55,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:55,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:55,890 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:55,890 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:55,896 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:56,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:57,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:58,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:23:58,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:23:58,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:00,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:00,451 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:00,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:00,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:00,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:00,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:01,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:05,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:05,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:05,686 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:05,686 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:05,692 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:06,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:07,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:09,772 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:24:10,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:10,433 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:10,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:10,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:10,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:11,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:11,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:13,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:24:13,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:24:13,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:16,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:18,852 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:18,853 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:18,853 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:18,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:19,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:19,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:21,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:24,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:24,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:24,195 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:24,201 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:24,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:25,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:27,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:28,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:24:28,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:24:28,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:29,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:29,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:29,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:29,017 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:29,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:29,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:33,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:33,919 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:33,920 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:33,920 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:33,926 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:34,648 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:35,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:38,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:38,398 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:38,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:38,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:38,404 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:38,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:39,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:39,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:24:42,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:42,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:42,926 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:42,931 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:24:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:43,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:43,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:24:43,663 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:43,663 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:48,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:48,478 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:48,478 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:48,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:48,484 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:48,669 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:49,672 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:53,191 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:53,192 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:53,192 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:53,198 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:53,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:53,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:54,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:58,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:24:58,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:24:58,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:59,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:59,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:59,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:59,696 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:59,701 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:00,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:01,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:04,738 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:04,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:04,970 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:04,970 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:04,976 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:05,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:05,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:09,780 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:25:09,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:11,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:11,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:11,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:11,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:11,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:11,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:13,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:25:13,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:25:13,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:25:15,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:15,736 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:15,737 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:15,737 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:15,744 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:16,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:17,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:20,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:22,070 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:22,071 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:22,071 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:22,077 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:22,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:23,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:26,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:28,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:25:28,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:25:28,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:25:28,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:28,679 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:28,679 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:28,685 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:28,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:29,730 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:31,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:33,959 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:33,960 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:33,960 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:33,967 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:34,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:35,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:36,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:38,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:38,830 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:38,830 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:38,837 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:39,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:39,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:39,779 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:25:42,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:25:43,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:25:43,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:25:47,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:47,993 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:47,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:47,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:47,999 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:48,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:49,759 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:53,033 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:56,511 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:56,512 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:56,512 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:56,518 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:56,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:57,771 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:58,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:25:58,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:25:58,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:25:58,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:02,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:02,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:02,654 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:02,660 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:02,776 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:03,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:04,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:08,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:08,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:08,055 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:08,060 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:08,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:09,784 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:26:09,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:09,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:12,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:12,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:12,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:12,734 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:12,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:13,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:26:13,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:26:13,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:26:13,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:15,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:18,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:18,401 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:18,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:18,407 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:18,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:19,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:21,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:23,191 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:23,192 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:23,192 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:23,198 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:23,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:25,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:27,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:28,329 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:28,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:28,335 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:28,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:26:28,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:26:28,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:26:28,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:29,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:32,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:32,950 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:32,950 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:32,950 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:32,956 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:33,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:33,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:37,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:39,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:26:40,563 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:40,564 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:40,565 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:40,570 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:40,832 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:41,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:26:43,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:26:43,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:26:43,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:45,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:45,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:45,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:45,128 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:45,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:47,844 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:49,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:49,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:49,742 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:49,742 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:49,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:49,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:51,850 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:53,884 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:53,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:53,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:53,891 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:54,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:54,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:55,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:58,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:26:58,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:26:58,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:27:00,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:02,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:02,148 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:02,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:02,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:02,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:03,867 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:06,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:07,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:07,254 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:07,254 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:07,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:07,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:09,785 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:27:09,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:11,787 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:13,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:27:13,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:27:13,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:27:15,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:15,467 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:15,467 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:15,474 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:15,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:17,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:17,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:19,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:19,962 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:19,962 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:19,968 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:20,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:21,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:22,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:25,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:25,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:25,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:25,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:25,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:27,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:28,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:27:28,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:27:28,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:27:28,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:30,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:30,459 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:30,459 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:30,465 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:30,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:31,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:34,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:34,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:34,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:34,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:34,739 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:34,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:35,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:39,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:39,488 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:39,488 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:39,494 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:39,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:27:39,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:39,919 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:41,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:27:43,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:27:43,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:27:44,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:44,857 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:44,858 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:44,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:44,863 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:44,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:45,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:49,867 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:49,868 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:49,868 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:49,868 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:49,874 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:49,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:51,938 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:54,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:54,614 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:54,614 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:54,620 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:54,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:55,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:55,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:58,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:27:58,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:27:58,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:00,503 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:00,503 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:00,503 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:00,509 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:00,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:01,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:01,953 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:04,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:04,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:04,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:04,780 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:04,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:05,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:06,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:09,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:28:11,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:13,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:13,309 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:13,309 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:13,315 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:13,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:28:13,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:28:13,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:13,967 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:15,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:17,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:18,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:18,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:18,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:18,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:18,976 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:19,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:22,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:23,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:23,185 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:23,185 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:23,191 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:23,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:25,988 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:27,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:27,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:27,778 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:27,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:27,784 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:27,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:28,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:28:28,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:28:28,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:29,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:32,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:32,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:32,299 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:32,305 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:32,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:33,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:34,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:36,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:36,571 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:36,572 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:36,578 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:37,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:38,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:38,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:39,792 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:28:43,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:28:43,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:28:43,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:43,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:45,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:45,304 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:45,304 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:45,310 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:46,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:48,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:49,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:49,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:49,638 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:49,638 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:49,644 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:50,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:52,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:54,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:57,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:57,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:57,442 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:57,448 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:58,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:58,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:28:58,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:28:58,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:59,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:00,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:02,864 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:02,865 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:02,865 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:02,871 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:03,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:04,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:04,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:07,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:07,404 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:07,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:07,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:08,047 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:09,794 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:29:10,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:10,796 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:11,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:11,667 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:11,667 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:11,673 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:12,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:13,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:29:13,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:29:13,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:29:14,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:16,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:16,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:16,212 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:16,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:16,217 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:17,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:18,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:21,104 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:21,105 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:21,106 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:21,112 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:22,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:22,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:22,112 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:25,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:25,930 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:25,930 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:25,936 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:26,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:27,936 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:28,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:28,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:29:28,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:29:28,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:29:31,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:31,384 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:31,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:31,390 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:32,084 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:33,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:34,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:35,911 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:35,912 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:35,912 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:35,918 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:36,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:38,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:38,919 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:39,799 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:29:40,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:40,538 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:40,538 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:40,544 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:41,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:42,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:43,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:29:43,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:29:43,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:29:44,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:45,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:45,275 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:45,276 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:45,281 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:46,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:48,109 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:49,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:49,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:49,534 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:49,540 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:50,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:50,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:52,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:55,482 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:55,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:55,483 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:55,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:56,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:56,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:58,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:58,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:29:58,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:29:58,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:30:01,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:04,047 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:04,048 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:04,048 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:04,054 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:04,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:06,135 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:07,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:09,798 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:30:10,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:10,339 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:10,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:10,345 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:11,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:12,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:12,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:13,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:30:13,412 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:30:13,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:30:16,272 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:16,272 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:16,273 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:16,279 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:17,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:18,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:18,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:21,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:21,184 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:21,184 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:21,191 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:22,159 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:22,159 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:24,191 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:27,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:27,316 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:27,316 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:27,322 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:28,165 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:28,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:30:28,412 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:30:28,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:30:29,708 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:30,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:34,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:34,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:34,077 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:34,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:34,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:35,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:36,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:39,804 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:30:40,805 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:42,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:42,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:42,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:42,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:42,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:43,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:30:43,412 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:30:43,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:30:44,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:46,703 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:48,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:48,131 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:48,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:48,137 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:48,195 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:50,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:52,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:53,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:53,681 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:53,681 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:53,687 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:54,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:56,209 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:57,688 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:58,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:30:58,412 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:30:58,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:31:00,499 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:00,499 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:00,499 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:00,506 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:01,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:02,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:03,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:05,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:05,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:05,213 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:05,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:05,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:06,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:09,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:09,803 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:31:10,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:10,064 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:10,064 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:10,070 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:10,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:12,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:13,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:31:13,412 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:31:13,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:31:14,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:15,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:15,200 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:15,200 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:15,207 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:15,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:16,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:20,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:23,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:23,224 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:23,224 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:23,230 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:23,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:24,250 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:26,231 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:28,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:28,111 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:28,111 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:28,116 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:28,254 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:28,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:31:28,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:31:28,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:31:30,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:31,262 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:33,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:33,381 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:33,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:33,388 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:34,264 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:36,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:36,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:38,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:38,142 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:38,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:38,148 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:38,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:39,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:31:40,274 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:41,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:43,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:43,763 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:43,763 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:43,769 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:43,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:31:43,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:31:43,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:31:44,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:46,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:47,283 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:52,225 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:52,225 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:52,226 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:52,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:52,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:53,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:54,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:56,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:56,653 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:56,653 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:56,659 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:57,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:58,301 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:58,659 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:58,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:31:58,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:31:58,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:32:03,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:03,315 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:03,315 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:03,321 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:04,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:04,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:04,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:08,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:08,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:08,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:08,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:09,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:09,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:32:09,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:10,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:13,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:13,422 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:13,422 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:13,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:13,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:32:13,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:32:13,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:32:14,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:14,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:15,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:17,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:17,973 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:17,974 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:17,979 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:18,328 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:20,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:21,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:24,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:24,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:24,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:24,339 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:24,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:26,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:26,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:28,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:28,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:28,639 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:28,646 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:28,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:32:28,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:32:28,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:32:29,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:30,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:32,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:33,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:33,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:33,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:33,156 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:33,353 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:34,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:38,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:39,809 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:32:41,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:41,634 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:41,634 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:41,640 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:42,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:43,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:43,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:32:43,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:32:43,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:32:44,368 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:47,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:47,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:47,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:47,016 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:47,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:48,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:49,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:51,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:51,971 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:51,971 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:51,978 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:52,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:54,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:54,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:58,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:32:58,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:32:58,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:33:00,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:00,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:00,552 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:00,552 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:00,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:01,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:02,395 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:05,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:05,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:05,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:05,133 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:05,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:06,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:06,400 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:09,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:09,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:09,482 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:09,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:09,815 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:33:10,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:10,407 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:11,816 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:13,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:33:13,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:33:13,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:33:14,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:14,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:14,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:14,347 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:14,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:16,415 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:17,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:21,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:21,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:21,256 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:21,261 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:21,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:22,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:23,262 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:26,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:26,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:26,494 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:26,500 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:27,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:28,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:28,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:28,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:33:28,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:33:28,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:33:31,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:31,611 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:31,611 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:31,617 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:32,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:33,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:34,442 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:35,884 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:35,885 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:35,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:35,891 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:36,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:38,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:38,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:39,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:33:40,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:40,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:40,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:40,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:40,450 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:42,454 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:43,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:33:43,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:33:43,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:33:44,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:44,707 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:44,708 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:44,708 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:44,714 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:45,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:46,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:49,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:51,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:51,492 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:51,493 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:51,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:52,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:52,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:55,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:55,290 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:55,290 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:55,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:55,296 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:55,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:56,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:58,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:33:58,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:33:58,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:01,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:01,095 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:01,095 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:01,096 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:01,102 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:01,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:02,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:06,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:09,604 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:09,605 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:09,605 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:09,611 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:09,819 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:34:10,491 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:11,820 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:12,496 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:13,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:13,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:13,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:13,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:13,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:34:13,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:34:13,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:14,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:16,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:17,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:17,947 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:17,948 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:17,948 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:17,954 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:18,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:20,508 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:22,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:24,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:24,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:24,878 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:24,883 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:25,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:26,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:28,884 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:28,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:34:28,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:34:28,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:29,718 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:29,719 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:29,719 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:29,725 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:30,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:32,527 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:34,761 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:37,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:37,522 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:37,522 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:37,530 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:37,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:38,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:39,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:34:39,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:42,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:42,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:42,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:42,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:42,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:43,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:34:43,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:34:43,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:44,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:45,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:46,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:46,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:46,465 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:46,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:46,545 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:48,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:50,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:50,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:50,874 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:50,874 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:50,880 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:51,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:52,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:55,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:58,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:34:58,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:34:58,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:59,220 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:59,253 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:59,253 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:59,259 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:59,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:00,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:01,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:04,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:04,021 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:04,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:04,028 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:04,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:06,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:07,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:09,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:09,015 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:09,015 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:09,021 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:09,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:09,823 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:35:10,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:12,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:13,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:35:13,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:35:13,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:35:15,516 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:15,517 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:15,517 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:15,523 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:15,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:16,589 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:18,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:21,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:21,016 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:21,016 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:21,022 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:21,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:22,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:24,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:26,054 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:26,055 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:26,055 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:26,061 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:26,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:28,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:28,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:35:28,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:35:28,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:35:29,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:30,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:30,863 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:30,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:30,870 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:31,612 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:32,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:34,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:36,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:36,090 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:36,090 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:36,096 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:36,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:38,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:39,822 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:35:40,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:42,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:42,230 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:42,230 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:42,236 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:42,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:43,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:35:43,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:35:43,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:35:44,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:46,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:46,759 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:46,759 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:46,759 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:46,765 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:47,655 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:48,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:51,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:51,108 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:51,108 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:51,118 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:51,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:52,118 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:52,674 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:55,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:55,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:55,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:55,156 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:55,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:56,682 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:57,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:58,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:35:58,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:35:58,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:01,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:01,712 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:01,712 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:01,718 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:02,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:02,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:04,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:07,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:07,166 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:07,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:07,172 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:07,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:08,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:08,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:09,824 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:36:11,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:11,882 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:11,882 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:11,888 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:12,699 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:13,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:13,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:36:13,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:36:13,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:14,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:18,054 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:18,055 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:18,055 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:18,060 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:18,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:19,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:20,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:23,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:23,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:23,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:23,018 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:23,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:24,715 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:25,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:28,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:28,144 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:28,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:28,150 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:28,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:28,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:36:28,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:36:28,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:30,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:30,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:34,042 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:34,042 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:34,042 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:34,049 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:34,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:36,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:36,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:38,259 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:38,260 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:38,260 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:38,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:38,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:39,826 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:36:40,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:41,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:43,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:43,506 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:43,507 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:43,512 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:43,738 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:43,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:36:43,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:36:43,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:44,741 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:47,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:49,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:49,393 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:49,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:49,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:49,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:50,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:52,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:54,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:54,101 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:54,101 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:54,108 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:54,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:56,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:58,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:58,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:36:58,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:36:58,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:59,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:59,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:59,288 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:59,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:59,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:00,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:03,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:04,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:04,372 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:04,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:04,378 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:04,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:06,769 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:08,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:08,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:08,464 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:08,465 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:08,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:08,771 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:09,829 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:37:10,776 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:13,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:13,471 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:13,472 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:13,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:13,477 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:13,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:13,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:37:13,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:37:13,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:37:14,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:18,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:18,368 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:18,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:18,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:18,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:19,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:20,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:22,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:22,400 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:22,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:22,406 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:22,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:24,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:25,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:27,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:27,867 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:27,867 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:27,873 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:28,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:28,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:37:28,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:37:28,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:37:30,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:31,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:32,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:32,396 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:32,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:32,402 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:32,808 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:34,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:36,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:39,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:39,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:39,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:39,161 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:39,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:39,831 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:37:40,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:41,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:43,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:37:43,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:37:43,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:37:44,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:44,685 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:44,685 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:44,691 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:44,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:46,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:47,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:49,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:49,442 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:49,442 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:49,448 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:49,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:50,832 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:53,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:56,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:56,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:56,038 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:56,044 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:56,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:58,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:58,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:37:58,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:37:58,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:37:59,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:01,178 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:01,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:01,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:01,184 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:01,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:02,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:05,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:06,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:06,673 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:06,673 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:06,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:06,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:08,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:09,833 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:38:10,835 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:11,686 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:11,687 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:11,687 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:11,693 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:11,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:12,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:13,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:38:13,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:38:13,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:38:16,256 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:16,257 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:16,257 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:16,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:16,263 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:16,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:18,864 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:20,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:20,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:20,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:20,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:20,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:21,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:22,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:26,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:26,826 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:26,826 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:26,827 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:26,833 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:26,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:28,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:28,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:38:28,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:38:28,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:38:30,930 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:30,930 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:30,930 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:30,937 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:31,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:31,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:32,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:36,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:39,838 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:38:39,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:39,846 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:39,846 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:39,852 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:39,889 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:40,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:42,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:43,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:38:43,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:38:43,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:38:45,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:45,412 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:45,412 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:45,418 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:45,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:46,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:48,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:50,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:50,051 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:50,051 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:50,057 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:50,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:52,908 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:54,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:54,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:54,310 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:54,310 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:54,316 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:54,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:56,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:58,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:38:58,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:38:58,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:38:59,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:02,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:02,552 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:02,552 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:02,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:02,917 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:04,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:04,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:07,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:07,802 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:07,803 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:07,808 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:07,924 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:08,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:09,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:09,837 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:39:12,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:12,383 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:12,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:12,390 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:12,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:13,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:39:13,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:39:13,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:39:14,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:15,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:16,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:16,769 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:16,770 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:16,775 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:16,936 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:18,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:20,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:21,823 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:21,824 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:21,824 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:21,829 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:21,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:22,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:25,830 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:26,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:26,363 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:26,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:26,370 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:26,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:28,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:28,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:39:28,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:39:28,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:39:31,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:31,259 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:31,259 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:31,259 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:31,265 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:31,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:32,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:36,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:36,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:36,974 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:36,974 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:36,980 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:37,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:38,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:39,842 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:39:41,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:41,237 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:41,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:41,243 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:41,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:42,244 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:42,974 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:43,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:39:43,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:39:43,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:39:45,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:45,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:45,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:45,129 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:45,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:46,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:48,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:51,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:51,375 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:51,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:51,381 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:51,984 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:52,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:53,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:55,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:55,922 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:55,922 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:55,927 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:55,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:56,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:58,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:58,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:39:58,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:39:58,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:40:01,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:01,741 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:01,741 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:01,747 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:01,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:03,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:04,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:08,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:08,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:08,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:08,128 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:09,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:09,841 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:40:09,843 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:11,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:13,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:13,601 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:13,602 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:13,607 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:13,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:40:13,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:40:13,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:40:14,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:15,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:15,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:18,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:18,217 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:18,218 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:18,223 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:19,022 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:20,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:21,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:22,939 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:22,940 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:22,940 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:22,946 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:23,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:25,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:25,947 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:28,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:28,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:28,458 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:28,464 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:28,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:40:28,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:40:28,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:40:29,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:31,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:31,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:35,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:35,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:35,196 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:35,202 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:36,047 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:36,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:37,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:39,847 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:40:41,755 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:41,756 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:41,756 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:41,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:41,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:42,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:43,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:43,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:40:43,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:40:43,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:40:46,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:46,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:46,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:46,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:47,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:47,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:49,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:52,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:53,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:53,215 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:53,215 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:53,221 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:54,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:55,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:58,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:58,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:40:58,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:40:58,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:41:01,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:01,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:01,202 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:01,208 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:02,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:03,086 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:03,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:05,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:05,939 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:05,939 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:05,945 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:06,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:07,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:08,946 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:09,846 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:41:10,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:10,791 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:10,791 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:10,797 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:11,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:13,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:13,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:41:13,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:41:13,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:41:14,219 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:14,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:14,858 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:14,858 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:14,864 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:15,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:17,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:19,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:19,847 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:19,847 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:19,847 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:19,853 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:20,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:21,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:24,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:26,568 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:26,569 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:26,570 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:26,576 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:27,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:28,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:41:28,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:41:28,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:41:29,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:30,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:31,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:31,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:31,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:31,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:32,128 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:33,131 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:35,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:36,256 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:36,257 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:36,258 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:36,263 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:37,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:39,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:39,848 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:41:40,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:41,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:41,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:41,535 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:41,541 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:42,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:43,147 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:43,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:41:43,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:41:43,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:41:46,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:47,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:47,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:47,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:47,135 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:47,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:49,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:52,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:52,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:52,252 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:52,252 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:52,257 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:53,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:55,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:56,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:56,955 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:56,955 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:56,961 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:57,162 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:57,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:58,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:41:58,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:41:58,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:41:59,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:01,939 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:01,940 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:01,940 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:01,946 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:02,169 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:02,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:03,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:06,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:06,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:06,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:06,780 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:07,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:08,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:09,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:09,850 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:42:11,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:11,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:11,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:11,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:12,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:13,187 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:13,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:42:13,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:42:13,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:42:14,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:18,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:18,308 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:18,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:18,314 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:19,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:19,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:21,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:23,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:23,019 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:23,020 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:23,025 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:23,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:25,026 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:25,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:28,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:28,507 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:28,507 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:28,513 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:28,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:42:28,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:42:28,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:42:29,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:30,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:31,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:33,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:33,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:33,696 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:33,702 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:34,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:35,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:35,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:38,491 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:38,492 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:38,493 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:38,498 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:39,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:39,853 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:42:40,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:41,226 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:43,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:43,891 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:43,891 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:43,897 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:43,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:42:43,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:42:43,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:42:44,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:45,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:46,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:48,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:48,780 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:48,781 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:48,787 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:49,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:51,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:51,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:54,230 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:54,230 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:54,231 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:54,236 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:54,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:57,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:57,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:58,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:58,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:58,608 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:58,614 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:58,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:42:58,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:42:58,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:42:59,251 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:01,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:02,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:03,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:03,834 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:03,834 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:03,840 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:04,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:05,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:07,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:08,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:08,402 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:08,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:08,408 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:09,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:09,855 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:43:11,267 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:13,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:13,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:43:13,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:43:13,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:43:14,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:14,510 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:14,510 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:14,516 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:15,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:17,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:19,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:19,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:19,711 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:19,711 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:19,717 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:20,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:21,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:24,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:24,582 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:24,582 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:24,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:24,588 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:25,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:27,292 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:28,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:43:28,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:43:28,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:43:29,074 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:29,221 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:29,221 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:29,227 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:29,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:30,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:31,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:35,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:36,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:36,479 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:36,479 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:36,484 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:37,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:39,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:39,857 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:43:40,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:41,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:41,171 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:41,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:41,177 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:41,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:43,316 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:43,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:43:43,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:43:43,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:43:46,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:50,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:50,158 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:50,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:50,164 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:50,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:51,328 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:52,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:54,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:54,634 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:54,634 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:54,640 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:55,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:57,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:57,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:58,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:43:58,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:43:58,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:43:59,761 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:59,762 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:59,762 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:59,767 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:00,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:01,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:02,805 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:05,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:05,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:05,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:05,376 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:06,349 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:07,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:08,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:09,862 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:44:10,980 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:10,981 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:10,981 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:10,987 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:11,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:13,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:13,987 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:13,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:44:13,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:44:13,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:44:19,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:19,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:19,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:19,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:19,355 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:19,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:21,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:23,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:23,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:23,767 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:23,773 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:24,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:24,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:25,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:28,531 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:28,532 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:28,532 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:28,538 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:28,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:44:28,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:44:28,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:44:29,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:30,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:31,385 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:33,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:33,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:33,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:33,184 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:33,388 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:35,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:36,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:37,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:37,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:37,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:37,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:38,395 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:39,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:39,861 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:44:41,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:43,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:44:43,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:44:43,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:44:45,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:45,499 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:45,499 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:45,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:46,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:47,409 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:47,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:50,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:50,614 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:50,614 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:50,620 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:51,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:52,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:53,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:55,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:55,792 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:55,792 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:55,798 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:56,420 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:57,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:57,798 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:58,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:44:58,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:44:58,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:45:03,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:04,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:04,239 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:04,239 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:04,245 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:04,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:05,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:08,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:08,955 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:08,956 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:08,956 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:08,961 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:09,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:09,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:45:11,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:13,935 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:13,936 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:13,936 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:13,942 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:13,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:45:13,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:45:13,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:45:14,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:14,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:15,449 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:18,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:18,386 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:18,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:18,392 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:18,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:19,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:21,458 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:24,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:26,264 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:26,265 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:26,266 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:26,271 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:26,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:27,467 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:28,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:45:28,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:45:28,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:45:30,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:31,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:31,284 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:31,284 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:31,290 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:31,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:33,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:35,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:35,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:35,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:35,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:35,935 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:36,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:37,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:39,865 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:45:40,866 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:42,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:42,348 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:42,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:42,354 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:42,488 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:43,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:45:43,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:45:43,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:45:45,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:46,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:47,031 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:47,032 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:47,032 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:47,037 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:47,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:49,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:52,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:52,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:52,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:52,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:52,521 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:53,503 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:55,508 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:56,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:56,378 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:56,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:56,384 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:56,508 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:57,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:58,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:45:58,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:45:58,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:45:59,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:00,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:00,829 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:00,829 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:00,835 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:01,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:02,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:03,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:05,991 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:05,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:05,992 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:05,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:06,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:07,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:07,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:09,867 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:46:10,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:10,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:10,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:10,716 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:11,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:13,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:13,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:13,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:46:13,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:46:13,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:46:15,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:15,911 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:15,911 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:15,917 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:16,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:17,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:18,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:21,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:21,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:21,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:21,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:21,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:23,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:24,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:26,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:26,187 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:26,187 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:26,193 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:26,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:27,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:28,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:46:28,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:46:28,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:46:29,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:30,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:30,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:30,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:30,701 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:31,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:33,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:34,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:35,755 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:35,755 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:35,756 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:35,761 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:36,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:37,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:39,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:39,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:46:40,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:40,046 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:40,046 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:40,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:40,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:41,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:43,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:46:43,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:46:43,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:46:45,219 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:47,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:47,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:47,065 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:47,071 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:47,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:49,588 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:51,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:51,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:51,765 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:51,765 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:51,771 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:52,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:53,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:56,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:58,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:46:58,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:46:58,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:47:00,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:00,326 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:00,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:00,332 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:00,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:01,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:02,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:05,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:05,338 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:05,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:05,344 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:05,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:07,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:08,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:09,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:09,531 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:09,531 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:09,536 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:09,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:09,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:47:11,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:13,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:13,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:47:13,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:47:13,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:47:15,906 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:15,907 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:15,907 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:15,913 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:16,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:17,628 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:18,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:20,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:20,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:20,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:20,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:20,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:23,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:24,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:25,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:25,979 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:25,979 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:25,985 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:26,640 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:27,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:28,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:47:28,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:47:28,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:47:30,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:30,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:30,613 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:30,613 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:30,619 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:30,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:33,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:35,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:35,652 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:35,653 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:35,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:35,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:35,659 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:37,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:39,877 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:47:40,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:40,215 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:40,215 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:40,221 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:40,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:41,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:41,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:43,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:47:43,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:47:43,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:47:45,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:45,573 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:45,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:45,580 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:45,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:46,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:47,675 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:50,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:50,382 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:50,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:50,388 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:50,677 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:51,680 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:52,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:57,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:58,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:58,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:58,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:58,340 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:58,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:58,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:47:58,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:47:59,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:47:59,690 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:03,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:03,089 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:03,089 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:03,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:03,095 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:03,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:05,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:08,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:08,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:08,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:08,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:08,164 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:08,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:09,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:09,876 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:48:12,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:12,791 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:12,792 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:12,797 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:13,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:13,798 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:13,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:48:13,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:48:14,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:48:15,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:18,299 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:18,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:18,300 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:18,305 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:18,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:19,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:19,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:24,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:24,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:24,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:24,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:24,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:25,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:25,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:28,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:48:28,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:48:29,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:48:29,188 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:29,267 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:29,268 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:29,273 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:29,733 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:30,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:31,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:35,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:38,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:38,380 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:38,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:38,386 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:38,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:39,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:39,882 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:48:40,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:43,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:48:43,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:48:44,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:48:46,266 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:46,952 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:46,952 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:46,953 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:46,958 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:47,759 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:49,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:51,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:52,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:52,276 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:52,277 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:52,282 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:52,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:53,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:56,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:56,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:56,807 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:56,814 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:57,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:57,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:58,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:48:58,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:48:59,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:48:59,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:01,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:01,120 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:01,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:01,126 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:01,780 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:03,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:03,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:07,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:07,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:07,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:07,017 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:07,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:09,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:09,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:09,881 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:49:11,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:11,386 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:11,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:11,392 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:11,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:13,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:13,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:49:13,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:49:14,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:49:14,279 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:15,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:15,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:15,664 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:15,670 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:15,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:17,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:19,706 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:20,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:20,577 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:20,577 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:20,585 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:20,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:21,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:25,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:25,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:25,933 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:25,934 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:25,940 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:26,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:27,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:28,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:49:28,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:49:29,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:49:31,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:31,739 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:31,740 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:31,740 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:31,746 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:31,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:33,831 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:36,782 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:38,507 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:38,507 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:38,508 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:38,514 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:38,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:39,838 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:39,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:49:41,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:43,147 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:43,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:43,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:43,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:43,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:49:43,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:49:44,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:49:45,844 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:47,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:50,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:50,281 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:50,281 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:50,287 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:50,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:51,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:52,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:54,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:54,681 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:54,681 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:54,687 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:54,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:57,688 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:57,863 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:58,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:49:58,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:49:59,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:50:02,544 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:02,545 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:02,545 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:02,551 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:02,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:03,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:03,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:08,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:09,885 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:50:10,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:10,764 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:10,764 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:10,770 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:10,879 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:13,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:13,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:13,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:50:13,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:50:14,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:50:14,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:14,961 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:14,962 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:14,967 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:15,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:17,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:19,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:22,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:22,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:22,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:22,520 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:22,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:23,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:24,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:28,220 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:28,221 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:28,221 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:28,228 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:28,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:28,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:50:28,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:50:29,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:50:29,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:30,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:33,971 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:33,972 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:33,972 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:33,978 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:34,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:35,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:36,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:39,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:50:41,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:41,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:41,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:41,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:41,178 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:41,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:43,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:43,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:50:43,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:50:44,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:50:46,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:47,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:47,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:47,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:47,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:47,932 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:49,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:51,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:52,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:52,236 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:52,236 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:52,242 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:52,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:53,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:56,759 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:56,760 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:56,760 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:56,761 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:56,766 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:56,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:58,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:50:58,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:50:59,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:50:59,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:01,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:01,126 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:01,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:01,133 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:01,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:02,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:03,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:05,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:05,507 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:05,507 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:05,513 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:05,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:07,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:07,962 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:09,889 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:51:11,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:11,624 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:11,624 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:11,630 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:11,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:12,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:13,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:13,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:51:13,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:51:14,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:51:15,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:15,557 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:15,557 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:15,563 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:15,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:17,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:18,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:19,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:19,653 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:19,654 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:19,659 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:19,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:21,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:23,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:25,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:25,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:25,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:25,446 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:25,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:27,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:28,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:51:29,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:51:29,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:51:29,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:29,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:29,765 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:29,765 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:29,771 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:29,992 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:31,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:34,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:34,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:35,000 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:35,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:35,006 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:36,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:38,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:39,891 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:51:39,894 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:41,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:41,402 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:41,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:41,408 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:42,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:43,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:51:44,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:51:44,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:51:44,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:45,291 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:46,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:46,415 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:46,415 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:46,421 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:47,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:48,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:50,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:50,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:50,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:50,879 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:50,885 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:51,022 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:54,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:55,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:58,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:51:59,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:51:59,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:51:59,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:59,451 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:59,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:59,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:00,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:01,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:02,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:04,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:04,132 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:04,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:04,138 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:05,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:06,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:07,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:09,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:52:10,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:10,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:10,223 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:10,229 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:11,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:12,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:12,230 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:13,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:52:14,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:52:14,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:52:15,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:15,139 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:15,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:15,145 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:16,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:18,061 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:18,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:21,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:21,626 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:21,627 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:21,633 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:22,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:23,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:24,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:26,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:26,084 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:26,084 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:26,090 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:27,073 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:28,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:29,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:52:29,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:52:29,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:52:29,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:32,712 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:32,712 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:32,712 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:32,718 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:33,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:34,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:34,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:38,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:38,626 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:38,627 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:38,632 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:39,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:39,896 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:52:39,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:40,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:44,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:52:44,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:52:44,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:52:44,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:44,634 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:44,634 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:44,641 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:45,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:45,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:46,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:48,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:48,484 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:48,485 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:48,490 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:49,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:50,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:51,491 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:52,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:52,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:52,787 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:52,793 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:53,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:54,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:56,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:57,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:57,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:57,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:57,890 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:58,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:59,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:52:59,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:52:59,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:53:00,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:02,225 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:02,226 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:02,226 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:02,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:02,232 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:03,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:04,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:06,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:06,502 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:06,502 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:06,507 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:07,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:07,508 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:08,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:09,901 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:53:10,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:10,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:10,654 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:10,659 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:11,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:12,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:12,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:14,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:53:14,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:53:14,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:53:16,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:16,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:16,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:16,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:17,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:18,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:18,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:20,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:20,672 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:20,673 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:20,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:21,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:22,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:23,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:26,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:26,088 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:26,089 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:26,095 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:26,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:27,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:29,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:53:29,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:53:29,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:53:29,247 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:31,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:31,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:31,312 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:31,318 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:32,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:33,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:34,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:36,991 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:36,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:36,992 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:36,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:37,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:39,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:39,900 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:53:39,902 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:42,731 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:42,732 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:42,732 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:42,738 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:43,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:43,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:44,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:53:44,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:53:44,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:53:45,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:48,479 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:48,480 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:48,480 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:48,486 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:49,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:49,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:50,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:53,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:53,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:53,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:53,527 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:54,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:55,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:55,527 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:59,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:53:59,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:53:59,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:54:00,531 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:00,532 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:00,532 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:00,532 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:00,538 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:01,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:01,272 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:05,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:05,723 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:05,724 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:05,724 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:05,730 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:06,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:07,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:09,906 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:54:10,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:10,373 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:10,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:10,379 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:11,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:11,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:11,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:14,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:54:14,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:54:14,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:54:15,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:15,229 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:15,229 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:15,236 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:15,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:17,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:17,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:21,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:21,986 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:21,986 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:21,991 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:22,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:22,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:23,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:26,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:26,491 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:26,491 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:26,497 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:27,303 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:27,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:28,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:29,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:54:29,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:54:29,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:54:30,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:30,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:30,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:30,984 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:31,166 WARNING FileStreamThread:2491270 [file_stream.py:request_with_retry():668] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/lilei_stones/hf-vlms/yrs4wcl6/file_stream. func: functools.partial(>, timeout=180.0), args: ('https://api.wandb.ai/files/lilei_stones/hf-vlms/yrs4wcl6/file_stream',), kwargs: {'json': {'files': {'wandb-summary.json': {'offset': 0, 'content': ['{"Finetune/Step": 4893, "Finetune/Loss": 0.8488271832466125, "Finetune/Loss (Raw)": 1.0013339519500732, "Finetune/Learning Rate": 1.7996355070443727e-07, "Finetune/Step Time": 5.442965777590871, "_timestamp": 1710280455.2282858, "_runtime": 26711.591413736343, "_step": 4893}']}, 'wandb-history.jsonl': {'offset': 4887, 'content': ['{"Finetune/Step": 4888, "Finetune/Loss": 0.8460896611213684, "Finetune/Loss (Raw)": 0.9415021538734436, "Finetune/Learning Rate": 1.8589386071896975e-07, "Finetune/Step Time": 5.447112096473575, "_timestamp": 1710280428.4794598, "_runtime": 26684.842587709427, "_step": 4888}', '{"Finetune/Step": 4889, "Finetune/Loss": 0.8469873666763306, "Finetune/Loss (Raw)": 0.8779363036155701, "Finetune/Learning Rate": 1.8470017954154262e-07, "Finetune/Step Time": 5.43055328913033, "_timestamp": 1710280433.5201728, "_runtime": 26689.88330078125, "_step": 4889}', '{"Finetune/Step": 4890, "Finetune/Loss": 0.8475115299224854, "Finetune/Loss (Raw)": 0.8610472083091736, "Finetune/Learning Rate": 1.8351030749034992e-07, "Finetune/Step Time": 5.449922828003764, "_timestamp": 1710280440.5315814, "_runtime": 26696.89470934868, "_step": 4890}', '{"Finetune/Step": 4891, "Finetune/Loss": 0.8467695713043213, "Finetune/Loss (Raw)": 0.7841209769248962, "Finetune/Learning Rate": 1.8232424502715672e-07, "Finetune/Step Time": 5.447011042386293, "_timestamp": 1710280445.7236629, "_runtime": 26702.086790800095, "_step": 4891}', '{"Finetune/Step": 4892, "Finetune/Loss": 0.8470671772956848, "Finetune/Loss (Raw)": 0.8847424983978271, "Finetune/Learning Rate": 1.811419926122493e-07, "Finetune/Step Time": 5.442794909700751, "_timestamp": 1710280450.372564, "_runtime": 26706.73569202423, "_step": 4892}', '{"Finetune/Step": 4893, "Finetune/Loss": 0.8488271832466125, "Finetune/Loss (Raw)": 1.0013339519500732, "Finetune/Learning Rate": 1.7996355070443727e-07, "Finetune/Step Time": 5.442965777590871, "_timestamp": 1710280455.2282858, "_runtime": 26711.591413736343, "_step": 4893}']}, 'wandb-events.jsonl': {'offset': 888, 'content': ['{"system.proc.memory.availableMB": 1783659.71, "system.memory": 13.6, "system.proc.memory.rssMB": 8293.02, "system.proc.memory.percent": 0.4, "system.cpu": 0.39, "system.cpu.0.cpu_percent": 18.31, "system.cpu.1.cpu_percent": 32.33, "system.cpu.2.cpu_percent": 0.3, "system.cpu.3.cpu_percent": 0.23, "system.cpu.4.cpu_percent": 1.07, "system.cpu.5.cpu_percent": 0.13, "system.cpu.6.cpu_percent": 0.03, "system.cpu.7.cpu_percent": 53.69, "system.cpu.8.cpu_percent": 2.33, "system.cpu.9.cpu_percent": 0.1, "system.cpu.10.cpu_percent": 0.0, "system.cpu.11.cpu_percent": 1.37, "system.cpu.12.cpu_percent": 0.03, "system.cpu.13.cpu_percent": 31.67, "system.cpu.14.cpu_percent": 1.33, "system.cpu.15.cpu_percent": 32.53, "system.cpu.16.cpu_percent": 17.61, "system.cpu.17.cpu_percent": 67.55, "system.cpu.18.cpu_percent": 0.1, "system.cpu.19.cpu_percent": 0.0, "system.cpu.20.cpu_percent": 0.03, "system.cpu.21.cpu_percent": 1.07, "system.cpu.22.cpu_percent": 0.0, "system.cpu.23.cpu_percent": 0.0, "system.cpu.24.cpu_percent": 14.87, "system.cpu.25.cpu_percent": 0.0, "system.cpu.26.cpu_percent": 0.63, "system.cpu.27.cpu_percent": 0.03, "system.cpu.28.cpu_percent": 0.53, "system.cpu.29.cpu_percent": 0.6, "system.cpu.30.cpu_percent": 0.0, "system.cpu.31.cpu_percent": 0.69, "system.cpu.32.cpu_percent": 6.03, "system.cpu.33.cpu_percent": 68.97, "system.cpu.34.cpu_percent": 0.03, "system.cpu.35.cpu_percent": 0.0, "system.cpu.36.cpu_percent": 0.0, "system.cpu.37.cpu_percent": 0.07, "system.cpu.38.cpu_percent": 0.03, "system.cpu.39.cpu_percent": 1.13, "system.cpu.40.cpu_percent": 24.97, "system.cpu.41.cpu_percent": 0.1, "system.cpu.42.cpu_percent": 0.17, "system.cpu.43.cpu_percent": 0.1, "system.cpu.44.cpu_percent": 0.1, "system.cpu.45.cpu_percent": 0.0, "system.cpu.46.cpu_percent": 0.07, "system.cpu.47.cpu_percent": 0.0, "system.cpu.48.cpu_percent": 0.03, "system.cpu.49.cpu_percent": 1.2, "system.cpu.50.cpu_percent": 0.0, "system.cpu.51.cpu_percent": 0.0, "system.cpu.52.cpu_percent": 0.0, "system.cpu.53.cpu_percent": 0.0, "system.cpu.54.cpu_percent": 0.0, "system.cpu.55.cpu_percent": 0.03, "system.cpu.56.cpu_percent": 0.13, "system.cpu.57.cpu_percent": 0.13, "system.cpu.58.cpu_percent": 0.2, "system.cpu.59.cpu_percent": 0.1, "system.cpu.60.cpu_percent": 0.07, "system.cpu.61.cpu_percent": 0.07, "system.cpu.62.cpu_percent": 0.03, "system.cpu.63.cpu_percent": 0.6, "system.cpu.64.cpu_percent": 0.0, "system.cpu.65.cpu_percent": 0.0, "system.cpu.66.cpu_percent": 0.0, "system.cpu.67.cpu_percent": 0.07, "system.cpu.68.cpu_percent": 0.03, "system.cpu.69.cpu_percent": 0.0, "system.cpu.70.cpu_percent": 0.0, "system.cpu.71.cpu_percent": 0.13, "system.cpu.72.cpu_percent": 0.0, "system.cpu.73.cpu_percent": 0.0, "system.cpu.74.cpu_percent": 0.0, "system.cpu.75.cpu_percent": 0.0, "system.cpu.76.cpu_percent": 0.0, "system.cpu.77.cpu_percent": 0.0, "system.cpu.78.cpu_percent": 0.1, "system.cpu.79.cpu_percent": 0.0, "system.cpu.80.cpu_percent": 0.0, "system.cpu.81.cpu_percent": 0.0, "system.cpu.82.cpu_percent": 0.0, "system.cpu.83.cpu_percent": 0.0, "system.cpu.84.cpu_percent": 0.67, "system.cpu.85.cpu_percent": 0.0, "system.cpu.86.cpu_percent": 0.33, "system.cpu.87.cpu_percent": 0.03, "system.cpu.88.cpu_percent": 0.03, "system.cpu.89.cpu_percent": 0.0, "system.cpu.90.cpu_percent": 0.03, "system.cpu.91.cpu_percent": 0.07, "system.cpu.92.cpu_percent": 0.0, "system.cpu.93.cpu_percent": 0.03, "system.cpu.94.cpu_percent": 0.07, "system.cpu.95.cpu_percent": 0.03, "system.cpu.96.cpu_percent": 0.0, "system.cpu.97.cpu_percent": 0.0, "system.cpu.98.cpu_percent": 0.0, "system.cpu.99.cpu_percent": 0.03, "system.cpu.100.cpu_percent": 0.0, "system.cpu.101.cpu_percent": 0.03, "system.cpu.102.cpu_percent": 0.0, "system.cpu.103.cpu_percent": 0.03, "system.cpu.104.cpu_percent": 0.0, "system.cpu.105.cpu_percent": 0.0, "system.cpu.106.cpu_percent": 0.0, "system.cpu.107.cpu_percent": 0.0, "system.cpu.108.cpu_percent": 0.0, "system.cpu.109.cpu_percent": 0.0, "system.cpu.110.cpu_percent": 0.6, "system.cpu.111.cpu_percent": 0.0, "system.cpu.112.cpu_percent": 0.0, "system.cpu.113.cpu_percent": 0.0, "system.cpu.114.cpu_percent": 0.0, "system.cpu.115.cpu_percent": 0.0, "system.cpu.116.cpu_percent": 0.0, "system.cpu.117.cpu_percent": 0.0, "system.cpu.118.cpu_percent": 0.0, "system.cpu.119.cpu_percent": 0.0, "system.cpu.120.cpu_percent": 0.03, "system.cpu.121.cpu_percent": 0.03, "system.cpu.122.cpu_percent": 0.03, "system.cpu.123.cpu_percent": 0.0, "system.cpu.124.cpu_percent": 0.0, "system.cpu.125.cpu_percent": 0.03, "system.cpu.126.cpu_percent": 0.0, "system.cpu.127.cpu_percent": 0.0, "system.cpu.128.cpu_percent": 14.41, "system.cpu.129.cpu_percent": 0.0, "system.cpu.130.cpu_percent": 14.1, "system.cpu.131.cpu_percent": 32.33, "system.cpu.132.cpu_percent": 1.27, "system.cpu.133.cpu_percent": 0.1, "system.cpu.134.cpu_percent": 68.31, "system.cpu.135.cpu_percent": 0.67, "system.cpu.136.cpu_percent": 0.1, "system.cpu.137.cpu_percent": 67.43, "system.cpu.138.cpu_percent": 0.8, "system.cpu.139.cpu_percent": 68.13, "system.cpu.140.cpu_percent": 1.97, "system.cpu.141.cpu_percent": 0.03, "system.cpu.142.cpu_percent": 0.63, "system.cpu.143.cpu_percent": 0.13, "system.cpu.144.cpu_percent": 0.1, "system.cpu.145.cpu_percent": 0.57, "system.cpu.146.cpu_percent": 0.03, "system.cpu.147.cpu_percent": 0.0, "system.cpu.148.cpu_percent": 0.0, "system.cpu.149.cpu_percent": 0.03, "system.cpu.150.cpu_percent": 0.03, "system.cpu.151.cpu_percent": 1.2, "system.cpu.152.cpu_percent": 0.07, "system.cpu.153.cpu_percent": 0.0, "system.cpu.154.cpu_percent": 0.0, "system.cpu.155.cpu_percent": 0.03, "system.cpu.156.cpu_percent": 0.03, "system.cpu.157.cpu_percent": 0.03, "system.cpu.158.cpu_percent": 0.0, "system.cpu.159.cpu_percent": 0.07, "system.cpu.160.cpu_percent": 0.0, "system.cpu.161.cpu_percent": 0.03, "system.cpu.162.cpu_percent": 0.03, "system.cpu.163.cpu_percent": 0.13, "system.cpu.164.cpu_percent": 0.13, "system.cpu.165.cpu_percent": 0.07, "system.cpu.166.cpu_percent": 0.0, "system.cpu.167.cpu_percent": 1.13, "system.cpu.168.cpu_percent": 0.0, "system.cpu.169.cpu_percent": 0.0, "system.cpu.170.cpu_percent": 0.0, "system.cpu.171.cpu_percent": 67.64, "system.cpu.172.cpu_percent": 0.67, "system.cpu.173.cpu_percent": 0.03, "system.cpu.174.cpu_percent": 0.63, "system.cpu.175.cpu_percent": 8.3, "system.cpu.176.cpu_percent": 0.0, "system.cpu.177.cpu_percent": 1.1, "system.cpu.178.cpu_percent": 0.17, "system.cpu.179.cpu_percent": 0.07, "system.cpu.180.cpu_percent": 0.03, "system.cpu.181.cpu_percent": 1.1, "system.cpu.182.cpu_percent": 0.0, "system.cpu.183.cpu_percent": 32.61, "system.cpu.184.cpu_percent": 0.6, "system.cpu.185.cpu_percent": 0.13, "system.cpu.186.cpu_percent": 2.49, "system.cpu.187.cpu_percent": 0.0, "system.cpu.188.cpu_percent": 0.07, "system.cpu.189.cpu_percent": 1.17, "system.cpu.190.cpu_percent": 67.41, "system.cpu.191.cpu_percent": 0.03, "system.cpu.192.cpu_percent": 0.0, "system.cpu.193.cpu_percent": 0.0, "system.cpu.194.cpu_percent": 0.0, "system.cpu.195.cpu_percent": 0.03, "system.cpu.196.cpu_percent": 0.0, "system.cpu.197.cpu_percent": 0.07, "system.cpu.198.cpu_percent": 0.0, "system.cpu.199.cpu_percent": 0.0, "system.cpu.200.cpu_percent": 0.0, "system.cpu.201.cpu_percent": 0.0, "system.cpu.202.cpu_percent": 0.0, "system.cpu.203.cpu_percent": 0.0, "system.cpu.204.cpu_percent": 0.0, "system.cpu.205.cpu_percent": 0.0, "system.cpu.206.cpu_percent": 0.0, "system.cpu.207.cpu_percent": 0.0, "system.cpu.208.cpu_percent": 0.0, "system.cpu.209.cpu_percent": 0.0, "system.cpu.210.cpu_percent": 0.0, "system.cpu.211.cpu_percent": 0.0, "system.cpu.212.cpu_percent": 0.0, "system.cpu.213.cpu_percent": 0.0, "system.cpu.214.cpu_percent": 0.0, "system.cpu.215.cpu_percent": 0.0, "system.cpu.216.cpu_percent": 0.0, "system.cpu.217.cpu_percent": 0.0, "system.cpu.218.cpu_percent": 0.43, "system.cpu.219.cpu_percent": 0.0, "system.cpu.220.cpu_percent": 0.03, "system.cpu.221.cpu_percent": 0.0, "system.cpu.222.cpu_percent": 0.0, "system.cpu.223.cpu_percent": 0.07, "system.cpu.224.cpu_percent": 0.0, "system.cpu.225.cpu_percent": 0.0, "system.cpu.226.cpu_percent": 0.0, "system.cpu.227.cpu_percent": 0.0, "system.cpu.228.cpu_percent": 0.0, "system.cpu.229.cpu_percent": 0.0, "system.cpu.230.cpu_percent": 0.0, "system.cpu.231.cpu_percent": 0.0, "system.cpu.232.cpu_percent": 0.0, "system.cpu.233.cpu_percent": 0.03, "system.cpu.234.cpu_percent": 0.53, "system.cpu.235.cpu_percent": 0.0, "system.cpu.236.cpu_percent": 0.03, "system.cpu.237.cpu_percent": 0.0, "system.cpu.238.cpu_percent": 0.0, "system.cpu.239.cpu_percent": 0.03, "system.cpu.240.cpu_percent": 0.0, "system.cpu.241.cpu_percent": 0.0, "system.cpu.242.cpu_percent": 0.0, "system.cpu.243.cpu_percent": 0.0, "system.cpu.244.cpu_percent": 0.0, "system.cpu.245.cpu_percent": 0.1, "system.cpu.246.cpu_percent": 0.0, "system.cpu.247.cpu_percent": 0.0, "system.cpu.248.cpu_percent": 0.0, "system.cpu.249.cpu_percent": 0.0, "system.cpu.250.cpu_percent": 0.0, "system.cpu.251.cpu_percent": 0.0, "system.cpu.252.cpu_percent": 0.03, "system.cpu.253.cpu_percent": 0.0, "system.cpu.254.cpu_percent": 0.0, "system.proc.cpu.threads": 45, "system.disk.\\\\.usagePercent": 20.9, "system.disk.\\\\.usageGB": 50.52, "system.disk.in": 6.73, "system.disk.out": 1017.14, "system.network.sent": 15543496764.73, "system.network.recv": 58448851231.4, "system.gpu.0.memoryAllocated": 98.15, "system.gpu.process.0.memoryAllocated": 98.15, "system.gpu.1.memoryAllocated": 82.61, "system.gpu.2.memoryAllocated": 93.26, "system.gpu.3.memoryAllocated": 92.26, "system.gpu.4.memoryAllocated": 97.2, "system.gpu.5.memoryAllocated": 77.66, "system.gpu.6.memoryAllocated": 92.48, "system.gpu.7.memoryAllocated": 71.01, "system.gpu.0.memoryAllocatedBytes": 84309114880.0, "system.gpu.process.0.memoryAllocatedBytes": 84309114880.0, "system.gpu.1.memoryAllocatedBytes": 70964936704.0, "system.gpu.2.memoryAllocatedBytes": 80112713728.0, "system.gpu.3.memoryAllocatedBytes": 79248687104.0, "system.gpu.4.memoryAllocatedBytes": 83493322752.0, "system.gpu.5.memoryAllocatedBytes": 66707718144.0, "system.gpu.6.memoryAllocatedBytes": 79439527936.0, "system.gpu.7.memoryAllocatedBytes": 60995076096.0, "system.gpu.0.memory": 39.73, "system.gpu.process.0.memory": 39.73, "system.gpu.1.memory": 35.6, "system.gpu.2.memory": 32.6, "system.gpu.3.memory": 33.93, "system.gpu.4.memory": 32.73, "system.gpu.5.memory": 33.33, "system.gpu.6.memory": 31.67, "system.gpu.7.memory": 30.47, "system.gpu.0.gpu": 99.8, "system.gpu.process.0.gpu": 99.8, "system.gpu.1.gpu": 99.93, "system.gpu.2.gpu": 99.93, "system.gpu.3.gpu": 99.93, "system.gpu.4.gpu": 99.87, "system.gpu.5.gpu": 99.93, "system.gpu.6.gpu": 99.93, "system.gpu.7.gpu": 99.93, "system.gpu.0.temp": 75.13, "system.gpu.process.0.temp": 75.13, "system.gpu.1.temp": 64.73, "system.gpu.2.temp": 64.0, "system.gpu.3.temp": 73.73, "system.gpu.4.temp": 74.73, "system.gpu.5.temp": 64.33, "system.gpu.6.temp": 64.0, "system.gpu.7.temp": 72.07, "system.gpu.0.powerWatts": 379.86, "system.gpu.process.0.powerWatts": 379.86, "system.gpu.1.powerWatts": 371.56, "system.gpu.2.powerWatts": 349.11, "system.gpu.3.powerWatts": 342.07, "system.gpu.4.powerWatts": 365.87, "system.gpu.5.powerWatts": 353.54, "system.gpu.6.powerWatts": 339.52, "system.gpu.7.powerWatts": 363.3, "system.gpu.0.powerPercent": 96.78, "system.gpu.process.0.powerPercent": 96.78, "system.gpu.1.powerPercent": 93.13, "system.gpu.2.powerPercent": 90.64, "system.gpu.3.powerPercent": 88.17, "system.gpu.4.powerPercent": 87.55, "system.gpu.5.powerPercent": 90.27, "system.gpu.6.powerPercent": 87.66, "system.gpu.7.powerPercent": 90.63, "_wandb": true, "_timestamp": 1710280449.900746, "_runtime": 26706.263874}']}, 'output.log': {'offset': 1, 'content': ['ERROR 2024-03-12T21:54:17.222823 =>> [Global Step] 004893 =>> LR :: 0.000000 -- Loss :: 0.8488: 94%|██████████████████████████████████████████▎ | 4893/5198 [7:24:58<26:26, 5.20s/it]\n']}}, 'dropped': 0}} +2024-03-12 21:54:31,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:33,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:34,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:36,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:36,180 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:36,181 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:36,186 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:36,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:37,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:39,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:39,905 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:54:42,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:42,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:42,322 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:42,328 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:43,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:43,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:44,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:54:44,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:54:44,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:54:44,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:47,264 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:47,265 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:47,265 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:47,271 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:47,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:49,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:49,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:51,714 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:51,715 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:51,715 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:51,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:52,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:53,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:54,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:55,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:55,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:55,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:55,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:56,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:57,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:59,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:54:59,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:54:59,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:55:00,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:00,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:00,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:00,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:00,814 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:01,344 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:01,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:05,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:05,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:05,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:05,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:05,283 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:05,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:07,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:09,907 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:55:10,798 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:10,805 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:10,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:10,805 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:10,810 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:11,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:11,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:14,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:55:14,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:55:14,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:55:15,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:15,967 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:15,967 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:15,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:15,973 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:16,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:17,362 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:20,747 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:20,748 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:20,748 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:20,754 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:21,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:21,368 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:21,754 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:26,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:26,125 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:26,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:26,131 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:26,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:27,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:27,373 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:29,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:55:29,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:55:29,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:55:32,299 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:32,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:32,833 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:32,833 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:32,839 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:33,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:33,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:37,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:37,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:37,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:37,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:37,984 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:38,385 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:39,388 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:39,909 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:55:42,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:42,236 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:42,237 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:42,243 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:42,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:43,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:43,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:44,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:55:44,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:55:44,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:55:46,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:46,511 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:46,511 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:46,516 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:47,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:47,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:48,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:51,816 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:51,817 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:51,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:51,823 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:52,402 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:53,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:53,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:57,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:57,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:57,767 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:57,773 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:58,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:59,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:55:59,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:55:59,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:55:59,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:59,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:04,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:06,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:06,084 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:06,084 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:06,090 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:06,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:07,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:09,911 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:56:09,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:11,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:11,346 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:11,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:11,353 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:11,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:13,427 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:14,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:56:14,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:56:14,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:56:15,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:16,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:16,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:16,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:16,170 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:16,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:17,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:21,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:24,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:24,685 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:24,685 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:24,691 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:25,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:25,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:26,692 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:29,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:56:29,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:56:29,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:56:29,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:29,248 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:29,248 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:29,255 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:29,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:31,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:32,256 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:33,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:33,641 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:33,641 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:33,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:34,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:35,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:37,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:37,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:37,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:37,946 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:37,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:38,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:39,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:39,913 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:56:42,915 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:43,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:43,893 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:43,893 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:43,899 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:44,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:56:44,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:56:44,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:56:44,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:45,467 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:48,296 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:48,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:48,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:48,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:48,527 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:49,472 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:49,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:53,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:53,098 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:53,099 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:53,104 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:53,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:54,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:55,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:58,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:58,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:58,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:58,268 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:58,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:59,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:56:59,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:56:59,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:56:59,268 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:59,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:02,908 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:02,908 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:02,908 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:02,914 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:03,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:03,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:04,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:07,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:07,217 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:07,218 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:07,223 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:07,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:09,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:09,916 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:57:10,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:12,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:12,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:12,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:12,322 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:12,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:13,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:14,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:57:14,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:57:14,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:57:16,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:20,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:20,082 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:20,083 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:20,089 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:20,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:21,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:22,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:24,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:24,586 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:24,586 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:24,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:25,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:25,516 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:27,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:28,880 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:28,881 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:28,881 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:28,887 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:29,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:57:29,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:57:29,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:57:29,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:29,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:33,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:33,745 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:33,746 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:33,746 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:33,751 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:34,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:35,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:38,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:38,190 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:38,190 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:38,196 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:38,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:39,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:39,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:39,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:57:44,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:57:44,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:57:44,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:57:44,268 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:44,728 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:44,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:44,729 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:44,734 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:45,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:45,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:49,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:49,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:49,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:49,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:49,439 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:49,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:51,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:54,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:54,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:54,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:54,293 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:54,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:55,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:55,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:59,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:57:59,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:57:59,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:58:00,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:00,239 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:00,239 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:00,245 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:00,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:01,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:01,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:04,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:04,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:04,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:04,346 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:04,560 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:05,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:06,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:09,920 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:58:10,774 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:10,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:10,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:10,780 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:11,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:11,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:11,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:14,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:58:14,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:58:14,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:58:17,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:19,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:19,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:19,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:19,100 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:19,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:19,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:23,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:23,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:23,703 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:23,703 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:23,709 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:24,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:25,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:28,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:28,410 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:28,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:28,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:28,416 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:28,588 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:29,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:58:29,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:58:29,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:58:29,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:34,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:34,197 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:34,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:34,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:34,204 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:34,597 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:35,600 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:39,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:39,823 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:39,823 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:39,824 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:39,829 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:39,925 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:58:40,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:41,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:44,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:58:44,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:58:44,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:58:44,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:44,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:44,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:44,458 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:44,463 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:44,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:45,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:49,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:49,090 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:49,090 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:49,099 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:49,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:49,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:50,099 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:54,482 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:54,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:54,483 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:54,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:54,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:55,488 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:55,630 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:59,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:58:59,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:58:59,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:58:59,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:59,470 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:59,470 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:59,475 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:59,633 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:00,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:01,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:04,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:04,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:04,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:04,028 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:04,640 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:05,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:06,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:08,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:08,403 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:08,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:08,409 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:08,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:09,650 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:09,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:59:11,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:14,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:59:14,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:59:14,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:59:14,543 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:14,543 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:14,543 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:14,550 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:14,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:15,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:17,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:19,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:19,239 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:19,239 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:19,245 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:19,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:19,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:23,246 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:24,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:24,554 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:24,554 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:24,560 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:24,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:25,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:28,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:29,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:59:29,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:59:29,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:59:29,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:29,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:29,439 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:29,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:29,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:31,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:34,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:39,482 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:39,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:59:41,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:41,036 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:41,037 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:41,042 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:41,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:41,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:44,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:59:44,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:59:44,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:59:45,264 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:46,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:46,061 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:46,062 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:46,067 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:46,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:47,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:50,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:50,008 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:50,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:50,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:50,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:51,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:51,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:55,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:55,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:55,288 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:55,293 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:55,715 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:55,715 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:56,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:59,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:59:59,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:59:59,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:00:01,706 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:01,706 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:01,707 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:01,707 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:01,713 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:01,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:03,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:06,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:09,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:00:10,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:10,896 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:10,897 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:10,902 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:11,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:11,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:11,903 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:14,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:00:14,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:00:14,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:00:15,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:15,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:15,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:15,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:15,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:17,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:17,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:20,507 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:20,508 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:20,508 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:20,513 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:20,747 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:21,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:22,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:24,932 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:24,933 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:24,933 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:24,939 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:25,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:25,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:27,939 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:29,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:00:29,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:00:29,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:00:29,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:29,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:29,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:29,889 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:30,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:31,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:33,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:34,575 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:34,576 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:34,576 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:34,582 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:34,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:35,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:39,086 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:39,086 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:39,087 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:39,087 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:39,092 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:39,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:39,778 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:39,931 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:00:43,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:43,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:43,791 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:43,797 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:44,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:00:44,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:00:44,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:00:44,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:44,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:45,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:49,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:49,840 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:49,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:49,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:49,846 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:50,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:51,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:54,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:54,676 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:54,676 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:54,676 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:54,682 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:54,796 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:55,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:59,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:00:59,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:00:59,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:01:00,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:00,912 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:00,913 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:00,913 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:00,918 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:01,808 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:01,808 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:05,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:06,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:06,133 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:06,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:06,139 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:06,811 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:07,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:09,934 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:01:11,935 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:12,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:12,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:12,186 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:12,192 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:12,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:13,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:14,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:01:14,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:01:14,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:01:17,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:18,550 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:18,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:18,550 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:18,556 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:18,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:19,834 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:22,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:23,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:23,026 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:23,026 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:23,032 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:23,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:23,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:28,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:29,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:01:29,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:01:29,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:01:31,304 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:31,304 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:31,304 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:31,310 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:31,850 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:31,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:33,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:36,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:36,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:36,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:36,100 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:36,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:37,860 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:39,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:39,936 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:01:40,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:40,729 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:40,730 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:40,735 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:40,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:41,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:43,972 WARNING FileStreamThread:2491270 [file_stream.py:request_with_retry():668] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/lilei_stones/hf-vlms/yrs4wcl6/file_stream. func: functools.partial(>, timeout=180.0), args: ('https://api.wandb.ai/files/lilei_stones/hf-vlms/yrs4wcl6/file_stream',), kwargs: {'json': {'files': {'wandb-summary.json': {'offset': 0, 'content': ['{"Finetune/Step": 4965, "Finetune/Loss": 0.8460050821304321, "Finetune/Loss (Raw)": 0.975489616394043, "Finetune/Learning Rate": 1.0515768749432942e-07, "Finetune/Step Time": 5.372652670368552, "_timestamp": 1710280839.0857449, "_runtime": 27095.44887280464, "_step": 4965}']}, 'wandb-history.jsonl': {'offset': 4958, 'content': ['{"Finetune/Step": 4959, "Finetune/Loss": 0.846284031867981, "Finetune/Loss (Raw)": 0.4407346248626709, "Finetune/Learning Rate": 1.1063312931768877e-07, "Finetune/Step Time": 5.411009738221765, "_timestamp": 1710280810.8957832, "_runtime": 27067.258911132812, "_step": 4959}', '{"Finetune/Step": 4960, "Finetune/Loss": 0.8468049764633179, "Finetune/Loss (Raw)": 0.8288559913635254, "Finetune/Learning Rate": 1.0971095892601702e-07, "Finetune/Step Time": 5.378879863768816, "_timestamp": 1710280815.4488149, "_runtime": 27071.81194281578, "_step": 4960}', '{"Finetune/Step": 4961, "Finetune/Loss": 0.8465644717216492, "Finetune/Loss (Raw)": 0.8159416317939758, "Finetune/Learning Rate": 1.0879262676243174e-07, "Finetune/Step Time": 5.377058727666736, "_timestamp": 1710280820.507151, "_runtime": 27076.870278835297, "_step": 4961}', '{"Finetune/Step": 4962, "Finetune/Loss": 0.8459833860397339, "Finetune/Loss (Raw)": 0.9207056164741516, "Finetune/Learning Rate": 1.0787813318331786e-07, "Finetune/Step Time": 5.375954560935497, "_timestamp": 1710280824.9324932, "_runtime": 27081.295621156693, "_step": 4962}', '{"Finetune/Step": 4963, "Finetune/Loss": 0.8452917337417603, "Finetune/Loss (Raw)": 0.8503878116607666, "Finetune/Learning Rate": 1.0696747854357259e-07, "Finetune/Step Time": 5.380342086777091, "_timestamp": 1710280829.883471, "_runtime": 27086.24659895897, "_step": 4963}', '{"Finetune/Step": 4964, "Finetune/Loss": 0.8451402187347412, "Finetune/Loss (Raw)": 0.947278618812561, "Finetune/Learning Rate": 1.0606066319660435e-07, "Finetune/Step Time": 5.372084315866232, "_timestamp": 1710280834.5755053, "_runtime": 27090.938633203506, "_step": 4964}', '{"Finetune/Step": 4965, "Finetune/Loss": 0.8460050821304321, "Finetune/Loss (Raw)": 0.975489616394043, "Finetune/Learning Rate": 1.0515768749432942e-07, "Finetune/Step Time": 5.372652670368552, "_timestamp": 1710280839.0857449, "_runtime": 27095.44887280464, "_step": 4965}']}, 'wandb-events.jsonl': {'offset': 901, 'content': ['{"system.proc.memory.availableMB": 1783514.09, "system.memory": 13.6, "system.proc.memory.rssMB": 8293.18, "system.proc.memory.percent": 0.4, "system.cpu": 0.39, "system.cpu.0.cpu_percent": 31.41, "system.cpu.1.cpu_percent": 31.41, "system.cpu.2.cpu_percent": 0.83, "system.cpu.3.cpu_percent": 0.27, "system.cpu.4.cpu_percent": 1.8, "system.cpu.5.cpu_percent": 31.02, "system.cpu.6.cpu_percent": 69.07, "system.cpu.7.cpu_percent": 68.63, "system.cpu.8.cpu_percent": 0.0, "system.cpu.9.cpu_percent": 0.0, "system.cpu.10.cpu_percent": 1.3, "system.cpu.11.cpu_percent": 0.63, "system.cpu.12.cpu_percent": 0.07, "system.cpu.13.cpu_percent": 30.93, "system.cpu.14.cpu_percent": 1.96, "system.cpu.15.cpu_percent": 31.57, "system.cpu.16.cpu_percent": 7.54, "system.cpu.17.cpu_percent": 68.59, "system.cpu.18.cpu_percent": 0.13, "system.cpu.19.cpu_percent": 0.03, "system.cpu.20.cpu_percent": 0.07, "system.cpu.21.cpu_percent": 0.6, "system.cpu.22.cpu_percent": 0.0, "system.cpu.23.cpu_percent": 0.0, "system.cpu.24.cpu_percent": 23.81, "system.cpu.25.cpu_percent": 0.0, "system.cpu.26.cpu_percent": 0.07, "system.cpu.27.cpu_percent": 0.07, "system.cpu.28.cpu_percent": 1.63, "system.cpu.29.cpu_percent": 1.13, "system.cpu.30.cpu_percent": 0.03, "system.cpu.31.cpu_percent": 0.6, "system.cpu.32.cpu_percent": 29.73, "system.cpu.33.cpu_percent": 70.33, "system.cpu.34.cpu_percent": 0.07, "system.cpu.35.cpu_percent": 0.0, "system.cpu.36.cpu_percent": 0.0, "system.cpu.37.cpu_percent": 0.1, "system.cpu.38.cpu_percent": 1.0, "system.cpu.39.cpu_percent": 0.23, "system.cpu.40.cpu_percent": 0.0, "system.cpu.41.cpu_percent": 0.77, "system.cpu.42.cpu_percent": 0.1, "system.cpu.43.cpu_percent": 0.1, "system.cpu.44.cpu_percent": 0.27, "system.cpu.45.cpu_percent": 0.0, "system.cpu.46.cpu_percent": 0.07, "system.cpu.47.cpu_percent": 7.34, "system.cpu.48.cpu_percent": 0.1, "system.cpu.49.cpu_percent": 1.33, "system.cpu.50.cpu_percent": 0.07, "system.cpu.51.cpu_percent": 0.07, "system.cpu.52.cpu_percent": 0.03, "system.cpu.53.cpu_percent": 0.07, "system.cpu.54.cpu_percent": 0.03, "system.cpu.55.cpu_percent": 0.07, "system.cpu.56.cpu_percent": 0.13, "system.cpu.57.cpu_percent": 0.07, "system.cpu.58.cpu_percent": 0.2, "system.cpu.59.cpu_percent": 0.03, "system.cpu.60.cpu_percent": 0.07, "system.cpu.61.cpu_percent": 0.03, "system.cpu.62.cpu_percent": 0.0, "system.cpu.63.cpu_percent": 0.57, "system.cpu.64.cpu_percent": 0.0, "system.cpu.65.cpu_percent": 0.0, "system.cpu.66.cpu_percent": 0.0, "system.cpu.67.cpu_percent": 0.13, "system.cpu.68.cpu_percent": 0.1, "system.cpu.69.cpu_percent": 0.0, "system.cpu.70.cpu_percent": 0.0, "system.cpu.71.cpu_percent": 0.03, "system.cpu.72.cpu_percent": 0.07, "system.cpu.73.cpu_percent": 0.07, "system.cpu.74.cpu_percent": 0.0, "system.cpu.75.cpu_percent": 0.07, "system.cpu.76.cpu_percent": 0.03, "system.cpu.77.cpu_percent": 0.03, "system.cpu.78.cpu_percent": 0.03, "system.cpu.79.cpu_percent": 0.0, "system.cpu.80.cpu_percent": 0.0, "system.cpu.81.cpu_percent": 0.03, "system.cpu.82.cpu_percent": 0.03, "system.cpu.83.cpu_percent": 0.07, "system.cpu.84.cpu_percent": 0.2, "system.cpu.85.cpu_percent": 0.03, "system.cpu.86.cpu_percent": 0.03, "system.cpu.87.cpu_percent": 0.03, "system.cpu.88.cpu_percent": 0.0, "system.cpu.89.cpu_percent": 0.0, "system.cpu.90.cpu_percent": 0.0, "system.cpu.91.cpu_percent": 0.07, "system.cpu.92.cpu_percent": 0.0, "system.cpu.93.cpu_percent": 0.03, "system.cpu.94.cpu_percent": 0.0, "system.cpu.95.cpu_percent": 0.03, "system.cpu.96.cpu_percent": 0.03, "system.cpu.97.cpu_percent": 0.03, "system.cpu.98.cpu_percent": 0.0, "system.cpu.99.cpu_percent": 0.03, "system.cpu.100.cpu_percent": 0.0, "system.cpu.101.cpu_percent": 0.0, "system.cpu.102.cpu_percent": 0.0, "system.cpu.103.cpu_percent": 0.0, "system.cpu.104.cpu_percent": 0.0, "system.cpu.105.cpu_percent": 0.03, "system.cpu.106.cpu_percent": 0.03, "system.cpu.107.cpu_percent": 0.0, "system.cpu.108.cpu_percent": 0.0, "system.cpu.109.cpu_percent": 0.03, "system.cpu.110.cpu_percent": 0.63, "system.cpu.111.cpu_percent": 0.0, "system.cpu.112.cpu_percent": 0.0, "system.cpu.113.cpu_percent": 0.07, "system.cpu.114.cpu_percent": 0.0, "system.cpu.115.cpu_percent": 0.0, "system.cpu.116.cpu_percent": 0.03, "system.cpu.117.cpu_percent": 0.03, "system.cpu.118.cpu_percent": 0.03, "system.cpu.119.cpu_percent": 0.0, "system.cpu.120.cpu_percent": 0.0, "system.cpu.121.cpu_percent": 0.0, "system.cpu.122.cpu_percent": 0.03, "system.cpu.123.cpu_percent": 0.03, "system.cpu.124.cpu_percent": 0.0, "system.cpu.125.cpu_percent": 0.0, "system.cpu.126.cpu_percent": 0.03, "system.cpu.127.cpu_percent": 0.0, "system.cpu.128.cpu_percent": 0.0, "system.cpu.129.cpu_percent": 0.03, "system.cpu.130.cpu_percent": 0.2, "system.cpu.131.cpu_percent": 0.0, "system.cpu.132.cpu_percent": 0.67, "system.cpu.133.cpu_percent": 0.2, "system.cpu.134.cpu_percent": 0.57, "system.cpu.135.cpu_percent": 0.17, "system.cpu.136.cpu_percent": 0.17, "system.cpu.137.cpu_percent": 68.46, "system.cpu.138.cpu_percent": 2.43, "system.cpu.139.cpu_percent": 68.56, "system.cpu.140.cpu_percent": 0.7, "system.cpu.141.cpu_percent": 0.07, "system.cpu.142.cpu_percent": 1.17, "system.cpu.143.cpu_percent": 0.13, "system.cpu.144.cpu_percent": 0.0, "system.cpu.145.cpu_percent": 1.17, "system.cpu.146.cpu_percent": 0.07, "system.cpu.147.cpu_percent": 0.0, "system.cpu.148.cpu_percent": 0.0, "system.cpu.149.cpu_percent": 0.0, "system.cpu.150.cpu_percent": 0.07, "system.cpu.151.cpu_percent": 1.09, "system.cpu.152.cpu_percent": 0.1, "system.cpu.153.cpu_percent": 0.03, "system.cpu.154.cpu_percent": 0.0, "system.cpu.155.cpu_percent": 0.03, "system.cpu.156.cpu_percent": 0.03, "system.cpu.157.cpu_percent": 0.07, "system.cpu.158.cpu_percent": 0.0, "system.cpu.159.cpu_percent": 0.1, "system.cpu.160.cpu_percent": 0.0, "system.cpu.161.cpu_percent": 0.1, "system.cpu.162.cpu_percent": 0.0, "system.cpu.163.cpu_percent": 0.17, "system.cpu.164.cpu_percent": 0.13, "system.cpu.165.cpu_percent": 0.03, "system.cpu.166.cpu_percent": 0.0, "system.cpu.167.cpu_percent": 0.6, "system.cpu.168.cpu_percent": 0.0, "system.cpu.169.cpu_percent": 0.0, "system.cpu.170.cpu_percent": 0.03, "system.cpu.171.cpu_percent": 69.02, "system.cpu.172.cpu_percent": 1.27, "system.cpu.173.cpu_percent": 0.03, "system.cpu.174.cpu_percent": 0.63, "system.cpu.175.cpu_percent": 0.0, "system.cpu.176.cpu_percent": 0.0, "system.cpu.177.cpu_percent": 0.63, "system.cpu.178.cpu_percent": 0.0, "system.cpu.179.cpu_percent": 0.07, "system.cpu.180.cpu_percent": 0.03, "system.cpu.181.cpu_percent": 1.1, "system.cpu.182.cpu_percent": 0.0, "system.cpu.183.cpu_percent": 33.51, "system.cpu.184.cpu_percent": 1.17, "system.cpu.185.cpu_percent": 0.1, "system.cpu.186.cpu_percent": 0.67, "system.cpu.187.cpu_percent": 1.23, "system.cpu.188.cpu_percent": 0.03, "system.cpu.189.cpu_percent": 1.3, "system.cpu.190.cpu_percent": 66.51, "system.cpu.191.cpu_percent": 0.03, "system.cpu.192.cpu_percent": 0.07, "system.cpu.193.cpu_percent": 0.0, "system.cpu.194.cpu_percent": 0.0, "system.cpu.195.cpu_percent": 0.03, "system.cpu.196.cpu_percent": 0.03, "system.cpu.197.cpu_percent": 0.0, "system.cpu.198.cpu_percent": 0.0, "system.cpu.199.cpu_percent": 0.0, "system.cpu.200.cpu_percent": 0.0, "system.cpu.201.cpu_percent": 0.0, "system.cpu.202.cpu_percent": 0.0, "system.cpu.203.cpu_percent": 0.0, "system.cpu.204.cpu_percent": 0.0, "system.cpu.205.cpu_percent": 0.0, "system.cpu.206.cpu_percent": 0.0, "system.cpu.207.cpu_percent": 0.0, "system.cpu.208.cpu_percent": 0.0, "system.cpu.209.cpu_percent": 0.0, "system.cpu.210.cpu_percent": 0.0, "system.cpu.211.cpu_percent": 0.0, "system.cpu.212.cpu_percent": 0.0, "system.cpu.213.cpu_percent": 0.0, "system.cpu.214.cpu_percent": 0.0, "system.cpu.215.cpu_percent": 0.0, "system.cpu.216.cpu_percent": 0.0, "system.cpu.217.cpu_percent": 0.0, "system.cpu.218.cpu_percent": 0.47, "system.cpu.219.cpu_percent": 0.0, "system.cpu.220.cpu_percent": 0.1, "system.cpu.221.cpu_percent": 0.0, "system.cpu.222.cpu_percent": 0.0, "system.cpu.223.cpu_percent": 0.1, "system.cpu.224.cpu_percent": 0.0, "system.cpu.225.cpu_percent": 0.0, "system.cpu.226.cpu_percent": 0.0, "system.cpu.227.cpu_percent": 0.0, "system.cpu.228.cpu_percent": 0.0, "system.cpu.229.cpu_percent": 0.0, "system.cpu.230.cpu_percent": 0.0, "system.cpu.231.cpu_percent": 0.0, "system.cpu.232.cpu_percent": 0.0, "system.cpu.233.cpu_percent": 0.03, "system.cpu.234.cpu_percent": 0.0, "system.cpu.235.cpu_percent": 0.0, "system.cpu.236.cpu_percent": 0.0, "system.cpu.237.cpu_percent": 0.0, "system.cpu.238.cpu_percent": 0.0, "system.cpu.239.cpu_percent": 0.0, "system.cpu.240.cpu_percent": 0.0, "system.cpu.241.cpu_percent": 0.0, "system.cpu.242.cpu_percent": 0.0, "system.cpu.243.cpu_percent": 0.0, "system.cpu.244.cpu_percent": 0.0, "system.cpu.245.cpu_percent": 0.0, "system.cpu.246.cpu_percent": 0.0, "system.cpu.247.cpu_percent": 0.0, "system.cpu.248.cpu_percent": 0.0, "system.cpu.249.cpu_percent": 0.0, "system.cpu.250.cpu_percent": 0.0, "system.cpu.251.cpu_percent": 0.0, "system.cpu.252.cpu_percent": 0.0, "system.cpu.253.cpu_percent": 0.0, "system.cpu.254.cpu_percent": 0.0, "system.proc.cpu.threads": 45, "system.disk.\\\\.usagePercent": 20.9, "system.disk.\\\\.usageGB": 50.52, "system.disk.in": 6.73, "system.disk.out": 1030.31, "system.network.sent": 15769502352.8, "system.network.recv": 59135337396.8, "system.gpu.0.memoryAllocated": 98.15, "system.gpu.process.0.memoryAllocated": 98.15, "system.gpu.1.memoryAllocated": 82.61, "system.gpu.2.memoryAllocated": 93.26, "system.gpu.3.memoryAllocated": 92.26, "system.gpu.4.memoryAllocated": 97.2, "system.gpu.5.memoryAllocated": 77.66, "system.gpu.6.memoryAllocated": 92.48, "system.gpu.7.memoryAllocated": 71.01, "system.gpu.0.memoryAllocatedBytes": 84309114880.0, "system.gpu.process.0.memoryAllocatedBytes": 84309114880.0, "system.gpu.1.memoryAllocatedBytes": 70964936704.0, "system.gpu.2.memoryAllocatedBytes": 80112713728.0, "system.gpu.3.memoryAllocatedBytes": 79248687104.0, "system.gpu.4.memoryAllocatedBytes": 83493322752.0, "system.gpu.5.memoryAllocatedBytes": 66707718144.0, "system.gpu.6.memoryAllocatedBytes": 79439527936.0, "system.gpu.7.memoryAllocatedBytes": 60995076096.0, "system.gpu.0.memory": 36.47, "system.gpu.process.0.memory": 36.47, "system.gpu.1.memory": 34.13, "system.gpu.2.memory": 32.2, "system.gpu.3.memory": 32.4, "system.gpu.4.memory": 30.87, "system.gpu.5.memory": 32.67, "system.gpu.6.memory": 30.47, "system.gpu.7.memory": 33.13, "system.gpu.0.gpu": 99.93, "system.gpu.process.0.gpu": 99.93, "system.gpu.1.gpu": 99.93, "system.gpu.2.gpu": 99.87, "system.gpu.3.gpu": 99.93, "system.gpu.4.gpu": 99.87, "system.gpu.5.gpu": 99.93, "system.gpu.6.gpu": 99.93, "system.gpu.7.gpu": 93.27, "system.gpu.0.temp": 74.53, "system.gpu.process.0.temp": 74.53, "system.gpu.1.temp": 63.73, "system.gpu.2.temp": 63.27, "system.gpu.3.temp": 72.13, "system.gpu.4.temp": 74.13, "system.gpu.5.temp": 63.6, "system.gpu.6.temp": 63.2, "system.gpu.7.temp": 72.27, "system.gpu.0.powerWatts": 375.92, "system.gpu.process.0.powerWatts": 375.92, "system.gpu.1.powerWatts": 354.62, "system.gpu.2.powerWatts": 375.65, "system.gpu.3.powerWatts": 355.77, "system.gpu.4.powerWatts": 365.66, "system.gpu.5.powerWatts": 338.48, "system.gpu.6.powerWatts": 355.63, "system.gpu.7.powerWatts": 368.22, "system.gpu.0.powerPercent": 93.57, "system.gpu.process.0.powerPercent": 93.57, "system.gpu.1.powerPercent": 92.76, "system.gpu.2.powerPercent": 93.74, "system.gpu.3.powerPercent": 89.83, "system.gpu.4.powerPercent": 91.57, "system.gpu.5.powerPercent": 84.36, "system.gpu.6.powerPercent": 89.18, "system.gpu.7.powerPercent": 92.47, "_wandb": true, "_timestamp": 1710280839.929379, "_runtime": 27096.292507}']}, 'output.log': {'offset': 1, 'content': ['ERROR 2024-03-12T22:00:39.608796 =>> [Global Step] 004965 =>> LR :: 0.000000 -- Loss :: 0.8460: 96%|██████████████████████████████████████████▉ | 4965/5198 [7:31:22<19:06, 4.92s/it]\n']}}, 'dropped': 0}} +2024-03-12 22:01:44,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:01:44,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:01:44,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:01:44,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:48,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:48,152 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:48,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:48,159 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:48,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:49,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:50,193 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:53,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:53,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:53,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:53,304 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:53,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:53,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:55,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:58,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:58,580 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:58,580 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:58,586 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:58,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:59,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:01:59,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:01:59,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:01:59,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:01,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:04,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:05,000 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:05,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:05,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:05,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:05,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:07,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:09,942 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:02:10,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:10,015 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:10,015 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:10,021 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:10,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:11,908 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:13,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:14,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:02:14,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:02:14,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:02:14,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:14,276 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:14,276 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:14,282 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:14,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:15,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:18,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:22,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:22,561 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:22,562 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:22,567 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:22,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:23,568 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:23,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:28,256 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:28,257 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:28,257 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:28,263 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:28,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:29,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:02:29,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:02:29,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:02:29,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:29,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:32,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:32,937 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:32,937 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:32,943 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:33,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:33,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:34,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:37,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:37,401 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:37,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:37,407 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:37,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:37,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:39,941 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:02:40,942 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:42,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:42,313 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:42,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:42,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:42,953 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:43,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:44,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:02:44,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:02:44,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:02:46,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:47,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:47,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:47,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:47,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:47,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:49,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:51,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:52,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:52,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:52,246 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:52,252 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:52,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:53,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:56,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:56,960 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:56,961 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:56,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:56,966 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:56,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:57,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:59,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:02:59,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:02:59,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:03:01,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:01,302 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:01,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:01,309 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:01,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:01,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:02,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:07,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:07,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:07,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:07,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:07,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:07,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:09,946 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:03:09,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:12,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:14,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:03:14,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:03:14,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:03:14,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:14,558 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:14,558 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:14,564 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:14,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:16,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:18,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:20,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:20,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:20,025 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:20,031 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:21,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:22,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:24,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:24,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:24,938 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:24,938 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:24,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:25,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:26,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:29,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:03:29,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:03:29,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:03:29,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:29,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:29,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:29,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:29,814 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:30,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:32,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:33,931 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:33,932 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:33,932 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:33,938 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:34,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:34,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:36,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:38,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:38,495 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:38,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:38,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:39,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:39,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:03:39,946 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:40,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:43,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:43,160 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:43,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:43,166 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:44,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:03:44,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:03:44,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:03:44,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:44,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:45,204 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:50,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:52,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:52,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:52,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:52,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:53,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:54,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:55,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:56,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:56,779 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:56,779 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:56,786 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:57,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:58,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:59,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:03:59,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:03:59,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:04:01,230 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:02,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:02,913 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:02,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:02,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:03,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:04,069 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:06,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:07,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:07,319 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:07,319 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:07,325 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:08,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:08,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:09,950 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:04:12,952 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:13,026 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:13,026 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:13,026 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:13,032 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:13,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:14,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:04:14,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:04:14,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:04:14,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:17,535 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:17,535 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:17,536 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:17,542 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:18,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:18,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:18,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:21,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:21,582 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:21,582 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:21,588 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:22,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:22,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:23,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:27,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:27,188 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:27,188 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:27,194 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:28,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:28,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:29,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:04:29,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:04:29,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:04:29,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:32,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:32,453 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:32,454 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:32,459 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:33,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:34,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:34,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:36,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:36,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:36,870 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:36,876 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:37,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:38,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:39,876 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:39,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:04:41,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:41,296 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:41,296 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:41,302 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:42,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:42,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:44,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:04:44,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:04:44,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:04:45,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:45,868 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:45,869 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:45,869 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:45,875 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:46,129 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:48,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:50,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:50,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:50,968 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:50,968 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:50,975 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:51,135 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:52,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:55,976 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:56,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:56,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:56,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:56,029 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:56,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:58,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:59,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:04:59,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:04:59,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:05:00,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:00,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:00,475 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:00,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:01,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:01,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:02,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:04,803 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:04,804 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:04,804 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:04,810 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:05,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:06,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:06,811 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:09,951 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:05:11,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:12,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:12,499 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:12,499 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:12,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:13,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:14,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:05:14,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:05:14,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:05:14,167 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:17,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:17,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:17,443 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:17,443 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:17,449 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:18,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:18,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:22,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:23,040 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:23,040 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:23,041 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:23,046 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:23,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:24,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:27,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:27,353 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:27,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:27,360 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:28,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:28,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:28,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:29,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:05:29,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:05:29,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:05:34,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:35,745 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:35,745 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:35,746 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:35,752 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:36,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:38,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:39,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:39,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:05:40,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:40,308 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:40,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:40,315 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:41,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:42,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:44,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:05:44,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:05:44,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:05:44,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:44,232 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:44,232 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:44,239 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:45,210 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:45,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:46,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:49,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:49,505 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:49,506 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:49,512 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:50,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:50,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:50,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:55,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:55,139 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:55,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:55,145 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:55,225 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:56,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:56,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:59,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:05:59,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:05:59,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:06:00,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:00,938 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:00,938 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:00,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:01,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:01,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:02,237 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:06,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:06,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:06,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:06,018 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:06,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:07,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:08,245 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:09,956 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:06:10,703 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:10,704 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:10,704 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:10,710 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:11,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:12,251 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:12,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:14,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:06:14,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:06:14,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:06:15,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:15,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:15,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:15,440 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:16,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:16,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:18,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:19,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:19,893 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:19,893 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:19,899 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:20,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:22,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:23,900 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:25,827 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:25,828 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:25,828 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:25,834 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:26,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:28,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:29,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:06:29,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:06:29,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:06:29,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:30,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:30,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:30,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:30,520 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:31,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:32,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:34,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:35,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:35,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:35,044 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:35,050 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:35,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:36,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:39,958 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:06:39,959 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:41,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:41,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:41,619 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:41,625 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:42,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:42,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:44,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:06:44,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:06:44,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:06:45,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:45,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:45,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:45,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:45,890 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:46,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:48,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:50,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:50,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:50,990 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:50,990 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:50,996 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:51,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:52,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:55,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:55,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:55,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:55,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:55,311 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:56,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:56,316 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:59,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:06:59,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:06:59,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:06:59,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:59,589 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:59,589 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:59,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:00,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:00,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:01,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:04,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:04,991 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:04,992 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:04,997 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:05,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:06,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:06,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:09,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:09,930 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:09,930 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:09,936 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:09,960 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:07:10,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:12,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:12,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:14,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:07:14,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:07:14,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:07:15,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:15,692 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:15,692 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:15,698 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:16,344 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:16,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:18,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:20,119 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:20,120 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:20,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:20,126 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:20,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:22,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:24,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:26,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:26,125 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:26,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:26,131 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:26,357 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:28,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:29,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:07:29,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:07:29,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:07:29,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:31,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:31,018 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:31,018 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:31,024 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:31,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:32,368 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:35,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:36,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:36,251 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:36,252 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:36,257 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:36,373 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:38,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:39,965 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:07:40,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:44,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:07:44,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:07:44,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:07:44,531 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:44,532 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:44,532 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:44,538 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:45,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:46,388 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:46,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:51,574 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:53,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:53,478 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:53,478 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:53,484 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:54,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:54,400 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:57,485 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:58,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:58,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:58,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:58,016 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:58,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:59,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:07:59,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:07:59,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:08:00,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:02,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:02,437 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:02,438 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:02,444 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:03,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:03,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:04,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:08,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:08,095 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:08,095 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:08,101 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:08,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:09,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:09,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:08:10,422 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:14,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:08:14,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:08:14,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:08:14,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:14,219 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:14,219 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:14,219 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:14,225 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:14,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:16,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:19,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:20,575 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:20,575 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:20,576 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:20,582 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:21,435 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:22,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:24,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:28,912 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:28,913 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:28,913 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:28,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:29,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:08:29,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:08:29,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:08:29,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:30,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:30,450 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:34,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:34,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:34,612 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:34,618 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:35,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:35,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:36,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:39,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:39,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:39,883 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:39,889 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:39,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:08:40,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:40,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:40,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:44,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:08:44,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:08:44,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:08:44,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:44,265 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:44,265 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:44,271 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:44,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:46,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:46,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:48,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:48,701 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:48,701 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:48,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:49,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:50,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:51,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:53,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:53,701 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:53,702 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:53,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:54,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:54,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:57,708 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:58,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:58,507 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:58,507 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:58,512 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:59,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:08:59,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:08:59,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:08:59,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:00,491 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:02,991 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:02,991 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:02,992 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:02,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:02,997 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:03,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:04,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:07,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:07,523 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:07,523 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:07,529 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:08,505 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:08,505 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:08,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:09,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:09:11,759 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:11,760 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:11,760 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:11,766 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:12,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:12,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:13,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:14,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:09:14,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:09:14,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:09:17,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:17,035 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:17,035 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:17,041 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:17,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:18,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:19,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:21,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:21,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:21,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:21,520 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:21,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:22,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:24,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:26,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:26,825 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:26,825 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:26,831 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:27,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:28,533 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:29,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:09:29,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:09:29,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:09:30,299 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:31,906 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:31,907 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:31,907 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:31,913 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:32,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:32,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:35,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:36,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:36,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:36,613 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:36,618 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:37,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:38,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:39,974 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:09:40,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:41,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:41,995 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:41,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:42,001 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:42,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:44,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:09:44,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:09:44,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:09:44,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:45,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:45,832 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:45,832 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:45,838 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:46,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:46,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:46,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:50,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:50,830 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:50,830 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:50,836 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:51,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:52,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:52,836 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:56,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:56,285 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:56,285 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:56,291 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:56,572 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:58,291 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:58,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:59,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:09:59,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:09:59,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:10:02,797 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:02,797 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:02,798 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:02,804 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:03,581 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:03,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:04,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:07,831 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:07,832 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:07,832 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:07,838 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:08,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:08,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:09,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:09,973 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:10:13,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:13,001 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:13,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:13,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:13,597 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:14,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:10:14,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:10:14,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:10:14,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:15,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:17,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:17,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:17,866 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:17,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:18,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:18,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:20,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:25,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:26,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:26,418 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:26,418 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:26,424 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:26,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:28,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:29,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:10:29,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:10:29,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:10:30,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:30,910 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:30,910 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:30,915 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:30,916 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:31,624 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:32,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:35,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:38,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:38,035 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:38,036 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:38,042 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:38,633 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:39,975 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:10:40,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:40,976 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:42,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:42,082 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:42,082 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:42,088 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:42,640 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:44,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:10:44,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:10:44,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:10:44,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:46,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:46,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:46,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:46,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:46,824 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:47,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:48,650 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:51,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:51,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:51,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:51,128 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:51,652 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:52,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:52,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:55,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:55,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:55,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:55,573 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:55,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:56,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:57,574 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:59,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:10:59,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:10:59,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:11:03,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:03,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:03,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:03,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:03,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:04,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:04,674 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:08,986 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:09,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:09,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:09,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:09,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:09,677 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:09,977 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:11:10,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:14,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:11:14,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:11:14,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:11:14,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:14,836 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:14,837 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:14,837 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:14,843 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:15,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:16,690 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:19,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:19,791 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:19,792 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:19,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:19,798 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:20,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:20,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:24,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:24,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:24,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:24,648 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:24,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:25,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:26,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:29,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:11:29,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:11:29,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:11:31,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:32,071 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:32,072 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:32,072 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:32,078 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:32,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:34,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:36,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:36,573 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:36,573 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:36,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:36,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:36,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:38,724 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:39,980 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:11:41,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:42,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:42,173 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:42,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:42,180 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:42,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:44,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:11:44,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:11:44,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:11:44,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:46,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:46,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:46,866 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:46,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:47,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:47,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:48,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:52,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:52,794 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:52,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:52,800 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:53,743 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:53,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:54,747 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:58,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:59,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:11:59,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:11:59,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:12:00,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:00,974 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:00,974 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:00,980 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:01,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:02,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:04,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:06,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:06,401 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:06,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:06,407 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:06,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:08,766 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:09,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:09,985 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:12:12,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:12,319 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:12,319 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:12,325 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:12,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:14,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:12:14,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:12:14,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:12:14,775 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:15,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:17,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:17,857 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:17,857 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:17,863 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:18,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:18,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:20,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:25,898 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:26,864 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:26,865 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:26,865 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:26,871 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:27,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:28,796 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:29,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:12:29,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:12:29,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:12:31,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:31,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:31,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:31,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:31,924 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:32,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:32,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:36,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:36,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:36,651 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:36,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:36,656 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:36,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:38,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:39,984 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:12:41,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:41,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:41,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:41,808 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:41,814 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:41,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:42,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:44,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:12:44,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:12:44,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:12:46,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:46,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:46,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:46,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:46,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:47,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:48,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:51,783 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:51,784 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:51,785 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:51,790 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:51,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:52,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:52,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:56,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:56,460 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:56,460 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:56,466 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:56,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:58,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:58,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:59,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:12:59,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:12:59,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:13:00,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:00,585 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:00,585 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:00,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:00,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:02,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:03,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:05,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:05,443 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:05,443 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:05,449 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:05,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:06,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:09,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:09,989 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:13:10,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:10,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:10,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:10,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:10,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:10,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:14,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:13:14,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:13:14,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:13:15,279 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:15,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:15,513 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:15,513 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:15,519 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:15,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:16,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:20,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:20,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:20,578 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:20,579 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:20,585 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:20,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:22,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:24,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:24,933 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:24,934 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:24,940 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:25,878 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:25,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:26,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:29,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:13:29,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:13:29,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:13:30,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:30,306 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:30,306 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:30,313 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:30,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:31,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:32,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:36,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:36,084 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:36,084 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:36,090 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:36,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:36,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:37,091 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:39,989 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:13:42,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:43,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:43,712 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:43,713 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:43,719 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:43,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:44,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:13:44,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:13:44,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:13:44,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:48,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:52,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:52,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:52,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:52,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:52,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:52,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:54,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:59,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:13:59,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:13:59,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:13:59,299 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:01,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:01,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:01,202 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:01,209 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:01,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:02,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:05,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:05,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:05,782 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:05,782 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:05,789 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:05,938 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:06,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:09,995 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:14:10,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:10,379 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:10,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:10,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:10,387 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:10,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:12,953 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:14,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:14:14,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:14:14,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:14:16,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:18,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:18,701 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:18,701 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:18,708 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:18,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:20,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:21,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:23,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:23,599 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:23,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:23,607 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:23,967 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:24,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:27,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:28,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:28,603 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:28,604 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:28,611 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:28,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:29,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:14:29,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:14:29,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:14:30,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:33,064 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:33,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:33,065 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:33,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:33,072 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:33,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:34,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:38,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:39,993 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:14:41,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:41,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:41,475 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:41,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:41,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:43,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:43,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:44,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:14:44,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:14:44,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:14:46,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:46,435 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:46,435 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:46,441 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:47,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:49,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:49,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:50,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:50,699 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:50,699 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:50,705 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:51,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:53,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:54,706 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:55,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:56,000 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:56,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:56,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:56,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:57,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:59,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:14:59,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:14:59,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:15:00,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:01,091 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:01,092 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:01,092 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:01,100 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:02,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:03,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:05,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:05,810 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:05,810 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:05,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:05,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:06,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:07,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:09,995 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:15:10,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:14,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:15:14,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:15:14,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:15:14,104 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:14,266 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:14,267 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:14,275 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:15,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:15,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:16,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:20,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:20,468 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:20,468 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:20,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:21,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:21,479 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:23,069 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:25,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:25,573 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:25,573 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:25,581 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:26,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:26,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:27,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:29,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:15:29,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:15:29,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:15:32,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:33,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:33,996 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:33,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:34,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:34,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:35,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:38,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:39,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:39,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:39,474 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:39,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:39,997 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:15:40,096 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:41,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:43,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:44,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:15:44,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:15:44,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:15:44,516 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:44,516 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:44,517 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:44,525 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:45,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:47,109 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:49,491 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:49,492 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:49,492 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:49,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:49,503 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:50,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:51,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:54,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:54,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:54,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:54,062 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:54,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:55,062 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:55,128 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:58,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:58,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:58,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:58,584 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:59,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:15:59,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:15:59,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:15:59,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:00,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:01,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:03,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:03,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:03,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:03,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:04,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:05,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:05,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:09,964 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:09,965 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:09,965 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:09,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:09,999 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:16:10,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:11,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:11,159 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:14,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:16:14,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:16:14,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:16:14,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:14,839 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:14,839 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:14,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:15,165 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:16,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:17,173 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:20,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:20,965 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:20,966 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:20,976 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:21,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:21,976 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:23,182 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:26,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:29,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:16:29,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:16:29,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:16:29,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:29,281 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:29,281 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:29,290 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:30,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:31,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:32,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:33,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:33,750 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:33,750 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:33,758 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:34,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:35,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:37,759 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:38,503 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:38,504 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:38,504 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:38,512 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:39,210 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:40,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:16:41,217 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:43,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:43,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:43,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:43,448 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:43,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:44,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:16:44,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:16:44,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:16:44,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:45,226 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:48,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:48,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:48,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:48,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:48,159 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:48,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:49,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:53,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:56,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:56,579 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:56,579 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:56,587 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:57,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:58,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:59,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:16:59,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:16:59,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:16:59,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:01,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:01,651 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:01,651 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:01,663 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:02,251 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:03,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:03,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:06,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:06,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:06,608 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:06,617 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:07,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:09,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:09,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:10,003 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:17:11,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:11,214 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:11,214 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:11,223 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:11,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:13,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:14,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:17:14,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:17:14,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:17:15,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:16,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:16,375 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:16,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:16,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:17,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:17,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:20,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:21,848 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:21,849 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:21,849 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:21,857 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:22,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:23,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:25,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:26,591 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:26,591 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:26,592 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:26,601 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:27,307 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:29,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:17:29,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:17:29,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:17:29,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:31,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:32,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:32,253 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:32,253 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:32,259 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:32,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:33,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:36,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:37,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:37,389 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:37,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:37,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:38,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:39,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:40,012 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:17:42,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:44,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:17:44,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:17:44,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:17:45,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:45,556 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:45,557 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:45,566 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:46,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:47,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:47,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:52,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:53,904 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:53,905 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:53,905 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:53,915 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:54,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:55,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:57,916 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:58,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:58,652 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:58,652 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:58,662 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:59,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:17:59,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:17:59,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:17:59,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:01,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:03,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:05,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:05,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:05,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:05,020 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:05,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:07,391 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:09,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:09,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:09,446 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:09,446 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:09,453 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:10,008 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:18:10,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:11,400 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:14,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:18:14,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:18:14,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:18:14,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:14,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:14,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:14,286 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:14,295 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:14,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:15,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:19,296 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:20,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:20,001 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:20,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:20,014 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:20,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:21,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:24,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:24,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:24,439 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:24,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:24,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:25,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:25,427 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:29,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:18:29,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:18:29,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:18:30,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:30,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:30,962 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:30,962 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:30,972 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:31,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:33,438 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:36,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:36,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:36,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:36,643 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:36,652 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:37,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:39,450 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:40,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:18:41,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:41,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:41,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:41,897 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:41,908 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:42,454 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:43,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:44,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:18:44,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:18:44,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:18:46,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:47,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:47,785 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:47,785 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:47,795 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:48,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:49,472 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:51,825 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:53,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:53,578 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:53,578 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:53,587 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:54,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:55,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:57,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:59,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:18:59,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:18:59,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:18:59,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:59,770 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:59,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:59,783 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:00,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:01,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:02,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:03,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:03,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:03,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:03,822 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:04,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:05,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:07,823 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:08,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:08,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:08,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:08,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:08,505 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:09,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:10,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:19:13,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:13,715 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:13,716 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:13,716 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:13,725 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:14,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:19:14,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:19:14,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:19:14,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:15,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:18,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:19,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:19,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:19,323 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:19,329 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:19,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:21,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:23,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:23,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:23,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:23,664 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:23,671 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:24,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:25,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:28,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:28,369 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:28,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:28,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:28,377 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:28,545 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:29,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:19:29,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:19:29,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:19:29,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:32,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:32,819 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:32,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:32,827 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:33,554 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:33,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:35,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:37,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:37,214 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:37,215 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:37,222 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:37,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:39,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:39,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:40,015 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:19:42,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:42,568 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:42,568 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:42,575 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:42,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:43,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:44,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:19:44,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:19:44,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:19:45,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:50,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:51,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:51,428 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:51,428 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:51,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:51,589 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:53,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:55,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:55,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:55,562 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:55,562 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:55,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:55,597 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:57,603 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:59,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:19:59,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:19:59,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:19:59,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:59,646 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:59,646 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:59,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:00,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:00,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:01,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:05,518 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:05,519 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:05,519 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:05,528 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:05,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:06,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:07,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:10,017 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:20:11,108 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:11,108 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:11,109 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:11,115 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:11,628 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:12,116 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:13,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:14,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:20:14,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:20:14,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:20:16,176 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:16,177 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:16,177 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:16,185 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:16,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:17,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:17,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:21,761 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:21,762 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:21,762 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:21,771 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:22,645 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:22,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:23,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:26,759 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:26,760 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:26,760 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:26,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:27,652 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:28,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:29,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:20:29,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:20:29,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:20:29,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:32,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:32,549 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:32,550 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:32,557 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:32,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:33,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:34,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:39,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:39,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:39,628 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:39,629 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:39,635 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:39,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:40,019 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:20:41,677 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:44,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:20:44,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:20:44,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:20:45,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:48,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:48,036 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:48,036 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:48,046 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:48,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:49,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:51,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:54,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:54,954 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:54,954 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:54,960 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:55,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:56,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:57,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:59,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:20:59,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:20:59,011 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:20:59,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:59,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:59,659 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:59,666 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:59,699 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:01,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:02,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:05,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:05,555 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:05,555 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:05,562 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:05,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:07,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:08,562 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:10,025 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:21:10,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:10,627 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:10,627 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:10,633 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:10,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:11,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:13,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:14,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:21:14,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:21:14,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:21:16,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:16,716 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:16,717 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:16,723 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:16,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:18,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:19,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:21,526 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:21,527 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:21,527 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:21,533 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:21,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:23,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:24,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:28,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:28,039 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:28,039 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:28,045 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:28,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:29,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:21:29,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:21:29,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:21:29,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:30,268 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:35,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:36,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:36,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:36,501 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:36,507 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:36,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:37,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:40,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:21:41,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:41,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:41,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:41,047 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:41,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:41,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:43,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:44,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:21:44,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:21:44,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:21:45,686 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:45,686 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:45,686 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:45,692 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:45,754 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:46,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:47,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:49,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:49,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:49,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:49,625 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:49,759 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:51,763 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:52,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:54,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:54,638 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:54,638 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:54,644 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:54,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:55,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:57,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:59,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:21:59,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:21:59,011 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:22:03,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:08,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:10,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:22:14,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:22:14,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:22:14,054 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:22:14,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:19,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:24,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:29,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:34,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:39,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:40,028 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:22:45,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:48,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:22:48,591 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:22:48,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:22:50,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:55,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:00,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:04,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:23:04,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:23:04,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:23:06,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:10,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:23:12,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:17,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:19,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:23:19,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:23:19,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:23:22,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:27,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:32,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:34,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:23:34,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:23:34,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:23:37,084 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:40,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:23:43,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:48,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:49,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:23:49,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:23:49,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:23:53,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:58,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:03,095 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:04,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:24:04,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:24:04,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:24:09,087 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:10,040 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:24:15,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:19,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:24:19,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:24:19,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:24:20,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:25,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:30,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:34,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:24:34,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:24:34,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:24:35,052 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:40,037 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:24:41,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:46,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:49,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:24:49,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:24:49,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:24:52,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:57,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:02,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:04,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:25:04,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:25:04,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:25:07,956 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:10,040 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:25:13,042 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:18,042 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:19,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:25:19,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:25:19,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:25:23,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:28,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:33,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:34,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:25:34,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:25:34,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:25:39,056 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:40,045 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:25:45,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:49,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:25:49,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:25:49,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:25:50,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:55,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:26:00,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:26:04,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:26:04,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:26:04,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:26:05,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:26:10,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:26:11,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:26:16,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:26:19,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:26:19,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:26:19,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:26:21,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:26:21,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: telemetry +2024-03-12 22:26:21,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: exit +2024-03-12 22:26:21,914 INFO SenderThread:2491270 [sender.py:send_exit():589] handling exit code: 0 +2024-03-12 22:26:21,914 INFO SenderThread:2491270 [sender.py:send_exit():591] handling runtime: 28633 +2024-03-12 22:26:21,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:26:21,921 INFO SenderThread:2491270 [sender.py:send_exit():597] send defer +2024-03-12 22:26:21,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:21,921 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 0 +2024-03-12 22:26:21,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:21,921 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-03-12 22:26:21,921 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 1 +2024-03-12 22:26:21,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:21,921 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 1 +2024-03-12 22:26:21,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:21,921 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-03-12 22:26:21,922 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 2 +2024-03-12 22:26:21,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:21,922 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 2 +2024-03-12 22:26:21,922 INFO HandlerThread:2491270 [system_monitor.py:finish():203] Stopping system monitor +2024-03-12 22:26:21,922 DEBUG SystemMonitor:2491270 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-12 22:26:21,922 INFO HandlerThread:2491270 [interfaces.py:finish():202] Joined cpu monitor +2024-03-12 22:26:21,922 DEBUG SystemMonitor:2491270 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-12 22:26:21,923 INFO HandlerThread:2491270 [interfaces.py:finish():202] Joined disk monitor +2024-03-12 22:26:22,402 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:26:23,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:26:23,535 INFO HandlerThread:2491270 [interfaces.py:finish():202] Joined gpu monitor +2024-03-12 22:26:23,536 INFO HandlerThread:2491270 [interfaces.py:finish():202] Joined memory monitor +2024-03-12 22:26:23,536 INFO HandlerThread:2491270 [interfaces.py:finish():202] Joined network monitor +2024-03-12 22:26:23,536 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 22:26:23,536 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:23,536 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-03-12 22:26:23,536 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 3 +2024-03-12 22:26:23,536 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:26:23,537 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 22:26:23,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:23,537 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 3 +2024-03-12 22:26:23,538 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:26:23,538 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:26:23,544 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:26:23,545 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:23,545 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-03-12 22:26:23,545 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 4 +2024-03-12 22:26:23,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:23,545 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 4 +2024-03-12 22:26:23,545 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:23,545 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-03-12 22:26:23,545 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 5 +2024-03-12 22:26:23,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:23,545 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 5 +2024-03-12 22:26:23,546 DEBUG SenderThread:2491270 [sender.py:send():382] send: summary +2024-03-12 22:26:23,551 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:26:23,551 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:23,551 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-03-12 22:26:23,551 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 6 +2024-03-12 22:26:23,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:23,551 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 6 +2024-03-12 22:26:23,551 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:23,551 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-03-12 22:26:23,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:26:23,774 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 7 +2024-03-12 22:26:23,774 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:23,774 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 7 +2024-03-12 22:26:23,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:23,774 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-03-12 22:26:23,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 22:26:24,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/config.yaml +2024-03-12 22:26:24,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:26:24,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:26:25,147 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 8 +2024-03-12 22:26:25,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 22:26:25,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:25,147 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 8 +2024-03-12 22:26:25,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:25,147 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-03-12 22:26:25,147 INFO SenderThread:2491270 [job_builder.py:build():296] Attempting to build job artifact +2024-03-12 22:26:25,148 INFO SenderThread:2491270 [job_builder.py:_get_source_type():426] is repo sourced job +2024-03-12 22:26:25,237 INFO SenderThread:2491270 [job_builder.py:build():402] adding wandb-job metadata file +2024-03-12 22:26:25,268 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 9 +2024-03-12 22:26:25,268 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:25,268 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 9 +2024-03-12 22:26:25,268 DEBUG SenderThread:2491270 [sender.py:send():382] send: artifact +2024-03-12 22:26:25,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:26:25,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 22:26:26,312 INFO SenderThread:2491270 [sender.py:send_artifact():1494] sent artifact job-https___github.com_TRI-ML_prismatic-vlms.git_scripts_pretrain.py - {'id': 'QXJ0aWZhY3Q6NzUzNTUzNzEx', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE0MzcwMzk1OA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NzUzNTUzNzEx', 'versionIndex': 7}}} +2024-03-12 22:26:26,312 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:26,312 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-03-12 22:26:26,313 INFO SenderThread:2491270 [dir_watcher.py:finish():358] shutting down directory watcher +2024-03-12 22:26:26,410 INFO SenderThread:2491270 [dir_watcher.py:finish():388] scan: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files +2024-03-12 22:26:26,410 INFO SenderThread:2491270 [dir_watcher.py:finish():402] scan save: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/config.yaml config.yaml +2024-03-12 22:26:26,410 INFO SenderThread:2491270 [dir_watcher.py:finish():402] scan save: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/conda-environment.yaml conda-environment.yaml +2024-03-12 22:26:26,410 INFO SenderThread:2491270 [dir_watcher.py:finish():402] scan save: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-metadata.json wandb-metadata.json +2024-03-12 22:26:26,411 INFO SenderThread:2491270 [dir_watcher.py:finish():402] scan save: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/requirements.txt requirements.txt +2024-03-12 22:26:26,411 INFO SenderThread:2491270 [dir_watcher.py:finish():402] scan save: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log output.log +2024-03-12 22:26:26,411 INFO SenderThread:2491270 [dir_watcher.py:finish():402] scan save: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json wandb-summary.json +2024-03-12 22:26:26,411 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 10 +2024-03-12 22:26:26,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 22:26:26,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:26,411 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 10 +2024-03-12 22:26:26,420 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:26,420 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-03-12 22:26:26,420 INFO SenderThread:2491270 [file_pusher.py:finish():172] shutting down file pusher +2024-03-12 22:26:26,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 22:26:26,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 22:26:27,146 INFO wandb-upload_1:2491270 [upload_job.py:push():131] Uploaded file runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/config.yaml +2024-03-12 22:26:27,341 INFO wandb-upload_3:2491270 [upload_job.py:push():131] Uploaded file runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:26:27,347 INFO wandb-upload_2:2491270 [upload_job.py:push():131] Uploaded file runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/requirements.txt +2024-03-12 22:26:27,354 INFO wandb-upload_4:2491270 [upload_job.py:push():131] Uploaded file runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:26:27,371 INFO wandb-upload_0:2491270 [upload_job.py:push():131] Uploaded file runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/conda-environment.yaml +2024-03-12 22:26:27,571 INFO Thread-11 (_thread_body):2491270 [sender.py:transition_state():617] send defer: 11 +2024-03-12 22:26:27,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:27,572 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 11 +2024-03-12 22:26:27,572 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:27,572 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-03-12 22:26:27,572 INFO SenderThread:2491270 [file_pusher.py:join():178] waiting for file pusher +2024-03-12 22:26:27,573 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 12 +2024-03-12 22:26:27,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:27,573 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 12 +2024-03-12 22:26:27,573 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:27,573 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-03-12 22:26:27,573 INFO SenderThread:2491270 [file_stream.py:finish():595] file stream finish called +2024-03-12 22:26:27,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 22:26:28,207 INFO SenderThread:2491270 [file_stream.py:finish():599] file stream finish is done +2024-03-12 22:26:28,207 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 13 +2024-03-12 22:26:28,207 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 22:26:28,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:28,208 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 13 +2024-03-12 22:26:28,208 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:28,208 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-03-12 22:26:28,208 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 14 +2024-03-12 22:26:28,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:28,208 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 14 +2024-03-12 22:26:28,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: final +2024-03-12 22:26:28,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: footer +2024-03-12 22:26:28,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:28,209 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-03-12 22:26:28,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 22:26:28,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 22:26:28,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:26:28,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: server_info +2024-03-12 22:26:28,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: server_info +2024-03-12 22:26:28,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: get_summary +2024-03-12 22:26:28,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: sampled_history +2024-03-12 22:26:28,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: job_info +2024-03-12 22:26:28,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: job_info +2024-03-12 22:26:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: shutdown +2024-03-12 22:26:28,402 INFO HandlerThread:2491270 [handler.py:finish():869] shutting down handler +2024-03-12 22:26:29,255 INFO WriterThread:2491270 [datastore.py:close():296] close: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/run-yrs4wcl6.wandb +2024-03-12 22:26:29,401 INFO SenderThread:2491270 [sender.py:finish():1572] shutting down sender +2024-03-12 22:26:29,401 INFO SenderThread:2491270 [file_pusher.py:finish():172] shutting down file pusher +2024-03-12 22:26:29,401 INFO SenderThread:2491270 [file_pusher.py:join():178] waiting for file pusher